{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a4d3fe3b",
   "metadata": {},
   "source": [
    "一般不使用PINN解决高维PDE问题，这是展示一下DeepBSDE方法。\n",
    "\n",
    "https://arxiv.org/pdf/1707.02568v2.pdf\n",
    "\n",
    "https://github.com/frankhan91/DeepBSDE/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c875544d",
   "metadata": {},
   "source": [
    "Black-Scholes 方程\n",
    "\n",
    "\n",
    "equ3.3Black-Scholes.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "08d7ae42",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from scipy.stats import multivariate_normal as normal\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import *\n",
    "import time\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "731760c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Y_compute(Layer):\n",
    "    def __init__(self,dimension):\n",
    "        self.dimension=dimension\n",
    "        self.y_init = tf.Variable(np.random.uniform(size=[1]))\n",
    "        self.grad_init = tf.Variable(\n",
    "            np.random.uniform(size=[1, self.dimension]))\n",
    "        super(Y_compute, self).__init__()\n",
    "\n",
    "    def call(self, inputs, **kwargs):\n",
    "        all_one_vec = tf.ones_like(tf.reduce_sum(tf.reduce_sum(inputs, axis=1), axis=1, keepdims=True))\n",
    "        y = all_one_vec * self.y_init\n",
    "        grad = tf.matmul(all_one_vec, self.grad_init)\n",
    "        return y,grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e584edf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BSDE():\n",
    "    def __init__(self):\n",
    "        self.dimension =100\n",
    "        self.T = 1.0\n",
    "        self.partion = 20\n",
    "        self.interval = self.T / self.partion\n",
    "        self.sqrt_interval = np.sqrt(self.interval)\n",
    "        self.time_stamp = np.arange(0, self.partion) * self.interval\n",
    "        self.x_init = np.zeros(self.dimension)\n",
    "        self.rate = 0.02  # interest rate R\n",
    "        self.delta = 2.0 / 3\n",
    "        self.gammah = 0.2\n",
    "        self.gammal = 0.02\n",
    "        self.mu_bar = 0.02\n",
    "        self.vh = 50.0\n",
    "        self.vl = 70.0\n",
    "        self.slope = (self.gammah - self.gammal) / (self.vh - self.vl)\n",
    "\n",
    "    # f in the equation\n",
    "    def f_equ(self, t, x, y, grad):\n",
    "        piecewise_linear = tf.nn.relu(\n",
    "            tf.nn.relu(y - self.vh) * self.slope + self.gammah - self.gammal) + self.gammal\n",
    "        return (-(1 - self.delta) * piecewise_linear - self.rate) * y\n",
    "\n",
    "    # Final\n",
    "    def g_final(self, t, x):\n",
    "        return tf.reduce_min(x, 1, keepdims=True)\n",
    "\n",
    "    # Data generator with motion of X\n",
    "    def sample(self, num_sample):\n",
    "        dw_sample = normal.rvs(size=[num_sample, self.dimension,\n",
    "                                     self.partion]) * self.sqrt_interval\n",
    "        x_sample = np.zeros([num_sample, self.dimension, self.partion + 1])\n",
    "        x_sample[:, :, 0] = np.ones([num_sample, self.dimension]) * self.x_init\n",
    "        for i in range(self.partion):\n",
    "            x_sample[:, :, i + 1] = x_sample[:, :, i] + dw_sample[:, :, i]\n",
    "\n",
    "        y_true=self.g_final(self.T, x_sample[:, :, -1])\n",
    "        return x_sample,dw_sample,y_true\n",
    "\n",
    "\n",
    "    def part(self,x,num_hid=110,dup=2):\n",
    "        for i in range(dup):\n",
    "            x=Dense(num_hid,activation='relu')(x)\n",
    "            x=BatchNormalization(\n",
    "                momentum=0.99,\n",
    "                epsilon=1e-6,\n",
    "                beta_initializer=tf.random_normal_initializer(0.0, stddev=0.1),\n",
    "                gamma_initializer=tf.random_uniform_initializer(0.1, 0.5)\n",
    "            )(x)\n",
    "        x = Dense(self.dimension, activation='relu')(x)\n",
    "        return x\n",
    "\n",
    "    def init_model(self,):\n",
    "        x1=Input(shape=(self.dimension,self.partion+1))\n",
    "        dw=Input(shape=(self.dimension,self.partion))\n",
    "\n",
    "        y,grad=Y_compute(self.dimension)(x1)\n",
    "\n",
    "        for t in range(0, self.partion-1):\n",
    "            y = y - self.interval * (\n",
    "                self.f_equ(self.time_stamp[t], x1[:, :, t], y, grad))\n",
    "            grad = self.part(x1[:, :, t+1])\n",
    "\n",
    "        # terminal time\n",
    "        out = y - self.interval * self.f_equ(self.time_stamp[-1], x1[:, :, -2], y, grad)\n",
    "\n",
    "        model=Model(inputs=[x1,dw],outputs=out)\n",
    "        model.compile(loss='mse',optimizer='sgd')\n",
    "        return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "33430f86",
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.keras.backend.set_floatx('float64')\n",
    "bsde=BSDE()\n",
    "\n",
    "model=bsde.init_model()\n",
    "x1_valid,x2_valid,y_valid = bsde.sample(256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bcfab29c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_3 (InputLayer)            [(None, 100, 21)]    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "y_compute_1 (Y_compute)         ((None, 1), (None, 1 101         input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_80 (TFOpLambda (None, 1)            0           y_compute_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_40 (TFOpLambda)      (None, 1)            0           tf.math.subtract_80[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_80 (TFOpLambda (None, 1)            0           tf.nn.relu_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_40 (TFOpLa (None, 1)            0           tf.math.multiply_80[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_81 (TFOpLambda (None, 1)            0           tf.__operators__.add_40[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_41 (TFOpLambda)      (None, 1)            0           tf.math.subtract_81[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_41 (TFOpLa (None, 1)            0           tf.nn.relu_41[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_81 (TFOpLambda (None, 1)            0           tf.__operators__.add_41[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_82 (TFOpLambda (None, 1)            0           tf.math.multiply_81[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_82 (TFOpLambda (None, 1)            0           tf.math.subtract_82[0][0]        \n",
      "                                                                 y_compute_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_83 (TFOpLambda (None, 1)            0           tf.math.multiply_82[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_83 (TFOpLambda (None, 1)            0           y_compute_1[0][0]                \n",
      "                                                                 tf.math.multiply_83[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_84 (TFOpLambda (None, 1)            0           tf.math.subtract_83[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_42 (TFOpLambda)      (None, 1)            0           tf.math.subtract_84[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_84 (TFOpLambda (None, 1)            0           tf.nn.relu_42[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_42 (TFOpLa (None, 1)            0           tf.math.multiply_84[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_85 (TFOpLambda (None, 1)            0           tf.__operators__.add_42[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_43 (TFOpLambda)      (None, 1)            0           tf.math.subtract_85[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_43 (TFOpLa (None, 1)            0           tf.nn.relu_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_85 (TFOpLambda (None, 1)            0           tf.__operators__.add_43[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_86 (TFOpLambda (None, 1)            0           tf.math.multiply_85[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_86 (TFOpLambda (None, 1)            0           tf.math.subtract_86[0][0]        \n",
      "                                                                 tf.math.subtract_83[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_87 (TFOpLambda (None, 1)            0           tf.math.multiply_86[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_87 (TFOpLambda (None, 1)            0           tf.math.subtract_83[0][0]        \n",
      "                                                                 tf.math.multiply_87[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_88 (TFOpLambda (None, 1)            0           tf.math.subtract_87[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_44 (TFOpLambda)      (None, 1)            0           tf.math.subtract_88[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_88 (TFOpLambda (None, 1)            0           tf.nn.relu_44[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_44 (TFOpLa (None, 1)            0           tf.math.multiply_88[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_89 (TFOpLambda (None, 1)            0           tf.__operators__.add_44[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_45 (TFOpLambda)      (None, 1)            0           tf.math.subtract_89[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_45 (TFOpLa (None, 1)            0           tf.nn.relu_45[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_89 (TFOpLambda (None, 1)            0           tf.__operators__.add_45[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_90 (TFOpLambda (None, 1)            0           tf.math.multiply_89[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_90 (TFOpLambda (None, 1)            0           tf.math.subtract_90[0][0]        \n",
      "                                                                 tf.math.subtract_87[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_91 (TFOpLambda (None, 1)            0           tf.math.multiply_90[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_91 (TFOpLambda (None, 1)            0           tf.math.subtract_87[0][0]        \n",
      "                                                                 tf.math.multiply_91[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_92 (TFOpLambda (None, 1)            0           tf.math.subtract_91[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_46 (TFOpLambda)      (None, 1)            0           tf.math.subtract_92[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_92 (TFOpLambda (None, 1)            0           tf.nn.relu_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_46 (TFOpLa (None, 1)            0           tf.math.multiply_92[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_93 (TFOpLambda (None, 1)            0           tf.__operators__.add_46[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_47 (TFOpLambda)      (None, 1)            0           tf.math.subtract_93[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_47 (TFOpLa (None, 1)            0           tf.nn.relu_47[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_93 (TFOpLambda (None, 1)            0           tf.__operators__.add_47[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_94 (TFOpLambda (None, 1)            0           tf.math.multiply_93[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_94 (TFOpLambda (None, 1)            0           tf.math.subtract_94[0][0]        \n",
      "                                                                 tf.math.subtract_91[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_95 (TFOpLambda (None, 1)            0           tf.math.multiply_94[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_95 (TFOpLambda (None, 1)            0           tf.math.subtract_91[0][0]        \n",
      "                                                                 tf.math.multiply_95[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_96 (TFOpLambda (None, 1)            0           tf.math.subtract_95[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_48 (TFOpLambda)      (None, 1)            0           tf.math.subtract_96[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_96 (TFOpLambda (None, 1)            0           tf.nn.relu_48[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_48 (TFOpLa (None, 1)            0           tf.math.multiply_96[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_97 (TFOpLambda (None, 1)            0           tf.__operators__.add_48[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_49 (TFOpLambda)      (None, 1)            0           tf.math.subtract_97[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_49 (TFOpLa (None, 1)            0           tf.nn.relu_49[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_97 (TFOpLambda (None, 1)            0           tf.__operators__.add_49[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_98 (TFOpLambda (None, 1)            0           tf.math.multiply_97[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_98 (TFOpLambda (None, 1)            0           tf.math.subtract_98[0][0]        \n",
      "                                                                 tf.math.subtract_95[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_99 (TFOpLambda (None, 1)            0           tf.math.multiply_98[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_99 (TFOpLambda (None, 1)            0           tf.math.subtract_95[0][0]        \n",
      "                                                                 tf.math.multiply_99[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_100 (TFOpLambd (None, 1)            0           tf.math.subtract_99[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_50 (TFOpLambda)      (None, 1)            0           tf.math.subtract_100[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_100 (TFOpLambd (None, 1)            0           tf.nn.relu_50[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_50 (TFOpLa (None, 1)            0           tf.math.multiply_100[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_101 (TFOpLambd (None, 1)            0           tf.__operators__.add_50[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_51 (TFOpLambda)      (None, 1)            0           tf.math.subtract_101[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_51 (TFOpLa (None, 1)            0           tf.nn.relu_51[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_101 (TFOpLambd (None, 1)            0           tf.__operators__.add_51[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_102 (TFOpLambd (None, 1)            0           tf.math.multiply_101[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_102 (TFOpLambd (None, 1)            0           tf.math.subtract_102[0][0]       \n",
      "                                                                 tf.math.subtract_99[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_103 (TFOpLambd (None, 1)            0           tf.math.multiply_102[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_103 (TFOpLambd (None, 1)            0           tf.math.subtract_99[0][0]        \n",
      "                                                                 tf.math.multiply_103[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_104 (TFOpLambd (None, 1)            0           tf.math.subtract_103[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_52 (TFOpLambda)      (None, 1)            0           tf.math.subtract_104[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_104 (TFOpLambd (None, 1)            0           tf.nn.relu_52[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_52 (TFOpLa (None, 1)            0           tf.math.multiply_104[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_105 (TFOpLambd (None, 1)            0           tf.__operators__.add_52[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_53 (TFOpLambda)      (None, 1)            0           tf.math.subtract_105[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_53 (TFOpLa (None, 1)            0           tf.nn.relu_53[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_105 (TFOpLambd (None, 1)            0           tf.__operators__.add_53[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_106 (TFOpLambd (None, 1)            0           tf.math.multiply_105[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_106 (TFOpLambd (None, 1)            0           tf.math.subtract_106[0][0]       \n",
      "                                                                 tf.math.subtract_103[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_107 (TFOpLambd (None, 1)            0           tf.math.multiply_106[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_107 (TFOpLambd (None, 1)            0           tf.math.subtract_103[0][0]       \n",
      "                                                                 tf.math.multiply_107[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_108 (TFOpLambd (None, 1)            0           tf.math.subtract_107[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_54 (TFOpLambda)      (None, 1)            0           tf.math.subtract_108[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_108 (TFOpLambd (None, 1)            0           tf.nn.relu_54[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_54 (TFOpLa (None, 1)            0           tf.math.multiply_108[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_109 (TFOpLambd (None, 1)            0           tf.__operators__.add_54[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_55 (TFOpLambda)      (None, 1)            0           tf.math.subtract_109[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_55 (TFOpLa (None, 1)            0           tf.nn.relu_55[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_109 (TFOpLambd (None, 1)            0           tf.__operators__.add_55[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_110 (TFOpLambd (None, 1)            0           tf.math.multiply_109[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_110 (TFOpLambd (None, 1)            0           tf.math.subtract_110[0][0]       \n",
      "                                                                 tf.math.subtract_107[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_111 (TFOpLambd (None, 1)            0           tf.math.multiply_110[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_111 (TFOpLambd (None, 1)            0           tf.math.subtract_107[0][0]       \n",
      "                                                                 tf.math.multiply_111[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_112 (TFOpLambd (None, 1)            0           tf.math.subtract_111[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_56 (TFOpLambda)      (None, 1)            0           tf.math.subtract_112[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_112 (TFOpLambd (None, 1)            0           tf.nn.relu_56[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_56 (TFOpLa (None, 1)            0           tf.math.multiply_112[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_113 (TFOpLambd (None, 1)            0           tf.__operators__.add_56[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_57 (TFOpLambda)      (None, 1)            0           tf.math.subtract_113[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_57 (TFOpLa (None, 1)            0           tf.nn.relu_57[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_113 (TFOpLambd (None, 1)            0           tf.__operators__.add_57[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_114 (TFOpLambd (None, 1)            0           tf.math.multiply_113[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_114 (TFOpLambd (None, 1)            0           tf.math.subtract_114[0][0]       \n",
      "                                                                 tf.math.subtract_111[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_115 (TFOpLambd (None, 1)            0           tf.math.multiply_114[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_115 (TFOpLambd (None, 1)            0           tf.math.subtract_111[0][0]       \n",
      "                                                                 tf.math.multiply_115[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_116 (TFOpLambd (None, 1)            0           tf.math.subtract_115[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_58 (TFOpLambda)      (None, 1)            0           tf.math.subtract_116[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_116 (TFOpLambd (None, 1)            0           tf.nn.relu_58[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_58 (TFOpLa (None, 1)            0           tf.math.multiply_116[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_117 (TFOpLambd (None, 1)            0           tf.__operators__.add_58[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_59 (TFOpLambda)      (None, 1)            0           tf.math.subtract_117[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_59 (TFOpLa (None, 1)            0           tf.nn.relu_59[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_117 (TFOpLambd (None, 1)            0           tf.__operators__.add_59[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_118 (TFOpLambd (None, 1)            0           tf.math.multiply_117[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_118 (TFOpLambd (None, 1)            0           tf.math.subtract_118[0][0]       \n",
      "                                                                 tf.math.subtract_115[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_119 (TFOpLambd (None, 1)            0           tf.math.multiply_118[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_119 (TFOpLambd (None, 1)            0           tf.math.subtract_115[0][0]       \n",
      "                                                                 tf.math.multiply_119[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_120 (TFOpLambd (None, 1)            0           tf.math.subtract_119[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_60 (TFOpLambda)      (None, 1)            0           tf.math.subtract_120[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_120 (TFOpLambd (None, 1)            0           tf.nn.relu_60[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_60 (TFOpLa (None, 1)            0           tf.math.multiply_120[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_121 (TFOpLambd (None, 1)            0           tf.__operators__.add_60[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_61 (TFOpLambda)      (None, 1)            0           tf.math.subtract_121[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_61 (TFOpLa (None, 1)            0           tf.nn.relu_61[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_121 (TFOpLambd (None, 1)            0           tf.__operators__.add_61[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_122 (TFOpLambd (None, 1)            0           tf.math.multiply_121[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_122 (TFOpLambd (None, 1)            0           tf.math.subtract_122[0][0]       \n",
      "                                                                 tf.math.subtract_119[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_123 (TFOpLambd (None, 1)            0           tf.math.multiply_122[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_123 (TFOpLambd (None, 1)            0           tf.math.subtract_119[0][0]       \n",
      "                                                                 tf.math.multiply_123[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_124 (TFOpLambd (None, 1)            0           tf.math.subtract_123[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_62 (TFOpLambda)      (None, 1)            0           tf.math.subtract_124[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_124 (TFOpLambd (None, 1)            0           tf.nn.relu_62[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_62 (TFOpLa (None, 1)            0           tf.math.multiply_124[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_125 (TFOpLambd (None, 1)            0           tf.__operators__.add_62[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_63 (TFOpLambda)      (None, 1)            0           tf.math.subtract_125[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_63 (TFOpLa (None, 1)            0           tf.nn.relu_63[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_125 (TFOpLambd (None, 1)            0           tf.__operators__.add_63[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_126 (TFOpLambd (None, 1)            0           tf.math.multiply_125[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_126 (TFOpLambd (None, 1)            0           tf.math.subtract_126[0][0]       \n",
      "                                                                 tf.math.subtract_123[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_127 (TFOpLambd (None, 1)            0           tf.math.multiply_126[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_127 (TFOpLambd (None, 1)            0           tf.math.subtract_123[0][0]       \n",
      "                                                                 tf.math.multiply_127[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_128 (TFOpLambd (None, 1)            0           tf.math.subtract_127[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_64 (TFOpLambda)      (None, 1)            0           tf.math.subtract_128[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_128 (TFOpLambd (None, 1)            0           tf.nn.relu_64[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_64 (TFOpLa (None, 1)            0           tf.math.multiply_128[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_129 (TFOpLambd (None, 1)            0           tf.__operators__.add_64[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_65 (TFOpLambda)      (None, 1)            0           tf.math.subtract_129[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_65 (TFOpLa (None, 1)            0           tf.nn.relu_65[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_129 (TFOpLambd (None, 1)            0           tf.__operators__.add_65[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_130 (TFOpLambd (None, 1)            0           tf.math.multiply_129[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_130 (TFOpLambd (None, 1)            0           tf.math.subtract_130[0][0]       \n",
      "                                                                 tf.math.subtract_127[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_131 (TFOpLambd (None, 1)            0           tf.math.multiply_130[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_131 (TFOpLambd (None, 1)            0           tf.math.subtract_127[0][0]       \n",
      "                                                                 tf.math.multiply_131[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_132 (TFOpLambd (None, 1)            0           tf.math.subtract_131[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_66 (TFOpLambda)      (None, 1)            0           tf.math.subtract_132[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_132 (TFOpLambd (None, 1)            0           tf.nn.relu_66[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_66 (TFOpLa (None, 1)            0           tf.math.multiply_132[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_133 (TFOpLambd (None, 1)            0           tf.__operators__.add_66[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_67 (TFOpLambda)      (None, 1)            0           tf.math.subtract_133[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_67 (TFOpLa (None, 1)            0           tf.nn.relu_67[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_133 (TFOpLambd (None, 1)            0           tf.__operators__.add_67[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_134 (TFOpLambd (None, 1)            0           tf.math.multiply_133[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_134 (TFOpLambd (None, 1)            0           tf.math.subtract_134[0][0]       \n",
      "                                                                 tf.math.subtract_131[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_135 (TFOpLambd (None, 1)            0           tf.math.multiply_134[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_135 (TFOpLambd (None, 1)            0           tf.math.subtract_131[0][0]       \n",
      "                                                                 tf.math.multiply_135[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_136 (TFOpLambd (None, 1)            0           tf.math.subtract_135[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_68 (TFOpLambda)      (None, 1)            0           tf.math.subtract_136[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_136 (TFOpLambd (None, 1)            0           tf.nn.relu_68[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_68 (TFOpLa (None, 1)            0           tf.math.multiply_136[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_137 (TFOpLambd (None, 1)            0           tf.__operators__.add_68[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_69 (TFOpLambda)      (None, 1)            0           tf.math.subtract_137[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_69 (TFOpLa (None, 1)            0           tf.nn.relu_69[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_137 (TFOpLambd (None, 1)            0           tf.__operators__.add_69[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_138 (TFOpLambd (None, 1)            0           tf.math.multiply_137[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_138 (TFOpLambd (None, 1)            0           tf.math.subtract_138[0][0]       \n",
      "                                                                 tf.math.subtract_135[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_139 (TFOpLambd (None, 1)            0           tf.math.multiply_138[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_139 (TFOpLambd (None, 1)            0           tf.math.subtract_135[0][0]       \n",
      "                                                                 tf.math.multiply_139[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_140 (TFOpLambd (None, 1)            0           tf.math.subtract_139[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_70 (TFOpLambda)      (None, 1)            0           tf.math.subtract_140[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_140 (TFOpLambd (None, 1)            0           tf.nn.relu_70[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_70 (TFOpLa (None, 1)            0           tf.math.multiply_140[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_141 (TFOpLambd (None, 1)            0           tf.__operators__.add_70[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_71 (TFOpLambda)      (None, 1)            0           tf.math.subtract_141[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_71 (TFOpLa (None, 1)            0           tf.nn.relu_71[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_141 (TFOpLambd (None, 1)            0           tf.__operators__.add_71[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_142 (TFOpLambd (None, 1)            0           tf.math.multiply_141[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_142 (TFOpLambd (None, 1)            0           tf.math.subtract_142[0][0]       \n",
      "                                                                 tf.math.subtract_139[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_143 (TFOpLambd (None, 1)            0           tf.math.multiply_142[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_143 (TFOpLambd (None, 1)            0           tf.math.subtract_139[0][0]       \n",
      "                                                                 tf.math.multiply_143[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_144 (TFOpLambd (None, 1)            0           tf.math.subtract_143[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_72 (TFOpLambda)      (None, 1)            0           tf.math.subtract_144[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_144 (TFOpLambd (None, 1)            0           tf.nn.relu_72[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_72 (TFOpLa (None, 1)            0           tf.math.multiply_144[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_145 (TFOpLambd (None, 1)            0           tf.__operators__.add_72[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_73 (TFOpLambda)      (None, 1)            0           tf.math.subtract_145[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_73 (TFOpLa (None, 1)            0           tf.nn.relu_73[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_145 (TFOpLambd (None, 1)            0           tf.__operators__.add_73[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_146 (TFOpLambd (None, 1)            0           tf.math.multiply_145[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_146 (TFOpLambd (None, 1)            0           tf.math.subtract_146[0][0]       \n",
      "                                                                 tf.math.subtract_143[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_147 (TFOpLambd (None, 1)            0           tf.math.multiply_146[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_147 (TFOpLambd (None, 1)            0           tf.math.subtract_143[0][0]       \n",
      "                                                                 tf.math.multiply_147[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_148 (TFOpLambd (None, 1)            0           tf.math.subtract_147[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_74 (TFOpLambda)      (None, 1)            0           tf.math.subtract_148[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_148 (TFOpLambd (None, 1)            0           tf.nn.relu_74[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_74 (TFOpLa (None, 1)            0           tf.math.multiply_148[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_149 (TFOpLambd (None, 1)            0           tf.__operators__.add_74[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_75 (TFOpLambda)      (None, 1)            0           tf.math.subtract_149[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_75 (TFOpLa (None, 1)            0           tf.nn.relu_75[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_149 (TFOpLambd (None, 1)            0           tf.__operators__.add_75[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_150 (TFOpLambd (None, 1)            0           tf.math.multiply_149[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_150 (TFOpLambd (None, 1)            0           tf.math.subtract_150[0][0]       \n",
      "                                                                 tf.math.subtract_147[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_151 (TFOpLambd (None, 1)            0           tf.math.multiply_150[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_151 (TFOpLambd (None, 1)            0           tf.math.subtract_147[0][0]       \n",
      "                                                                 tf.math.multiply_151[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_152 (TFOpLambd (None, 1)            0           tf.math.subtract_151[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_76 (TFOpLambda)      (None, 1)            0           tf.math.subtract_152[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_152 (TFOpLambd (None, 1)            0           tf.nn.relu_76[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_76 (TFOpLa (None, 1)            0           tf.math.multiply_152[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_153 (TFOpLambd (None, 1)            0           tf.__operators__.add_76[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_77 (TFOpLambda)      (None, 1)            0           tf.math.subtract_153[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_77 (TFOpLa (None, 1)            0           tf.nn.relu_77[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_153 (TFOpLambd (None, 1)            0           tf.__operators__.add_77[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_154 (TFOpLambd (None, 1)            0           tf.math.multiply_153[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_154 (TFOpLambd (None, 1)            0           tf.math.subtract_154[0][0]       \n",
      "                                                                 tf.math.subtract_151[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_155 (TFOpLambd (None, 1)            0           tf.math.multiply_154[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_155 (TFOpLambd (None, 1)            0           tf.math.subtract_151[0][0]       \n",
      "                                                                 tf.math.multiply_155[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_156 (TFOpLambd (None, 1)            0           tf.math.subtract_155[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_78 (TFOpLambda)      (None, 1)            0           tf.math.subtract_156[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_156 (TFOpLambd (None, 1)            0           tf.nn.relu_78[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_78 (TFOpLa (None, 1)            0           tf.math.multiply_156[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_157 (TFOpLambd (None, 1)            0           tf.__operators__.add_78[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.relu_79 (TFOpLambda)      (None, 1)            0           tf.math.subtract_157[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_79 (TFOpLa (None, 1)            0           tf.nn.relu_79[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_157 (TFOpLambd (None, 1)            0           tf.__operators__.add_79[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_158 (TFOpLambd (None, 1)            0           tf.math.multiply_157[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_158 (TFOpLambd (None, 1)            0           tf.math.subtract_158[0][0]       \n",
      "                                                                 tf.math.subtract_155[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_159 (TFOpLambd (None, 1)            0           tf.math.multiply_158[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "input_4 (InputLayer)            [(None, 100, 20)]    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_159 (TFOpLambd (None, 1)            0           tf.math.subtract_155[0][0]       \n",
      "                                                                 tf.math.multiply_159[0][0]       \n",
      "==================================================================================================\n",
      "Total params: 101\n",
      "Trainable params: 101\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1fea13f3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# tf.keras.utils.plot_model(model, 'model.png', show_shapes=True)\n",
    "\n",
    "# tf 和 keras 混用，报错：\n",
    "# AttributeError: 'tensorflow.python.framework.ops.EagerTensor' object has no attribute '_keras_history'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7d8d9477",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Gradients do not exist for variables ['Variable:0'] when minimizing the loss.\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['Variable:0'] when minimizing the loss.\n",
      "step:     0,    loss: 1.0133e+01, Y_init: 5.8570e-01,   elapsed time: 246\n",
      "step:   100,    loss: 2.5218e-01, Y_init: -2.0426e+00,   elapsed time: 251\n",
      "step:   200,    loss: 1.6849e-01, Y_init: -2.2797e+00,   elapsed time: 255\n",
      "step:   300,    loss: 1.6774e-01, Y_init: -2.2935e+00,   elapsed time: 259\n",
      "step:   400,    loss: 1.6776e-01, Y_init: -2.2930e+00,   elapsed time: 263\n",
      "step:   500,    loss: 1.6749e-01, Y_init: -2.3027e+00,   elapsed time: 267\n",
      "step:   600,    loss: 1.6755e-01, Y_init: -2.2998e+00,   elapsed time: 271\n",
      "step:   700,    loss: 1.6751e-01, Y_init: -2.3017e+00,   elapsed time: 275\n",
      "step:   800,    loss: 1.6747e-01, Y_init: -2.3039e+00,   elapsed time: 279\n",
      "step:   900,    loss: 1.6775e-01, Y_init: -2.2934e+00,   elapsed time: 283\n",
      "step:  1000,    loss: 1.6746e-01, Y_init: -2.3138e+00,   elapsed time: 287\n",
      "step:  1100,    loss: 1.6747e-01, Y_init: -2.3041e+00,   elapsed time: 291\n",
      "step:  1200,    loss: 1.6754e-01, Y_init: -2.3002e+00,   elapsed time: 295\n",
      "step:  1300,    loss: 1.6755e-01, Y_init: -2.2999e+00,   elapsed time: 299\n",
      "step:  1400,    loss: 1.6744e-01, Y_init: -2.3101e+00,   elapsed time: 303\n",
      "step:  1500,    loss: 1.6783e-01, Y_init: -2.2912e+00,   elapsed time: 307\n",
      "step:  1600,    loss: 1.6769e-01, Y_init: -2.2950e+00,   elapsed time: 311\n",
      "step:  1700,    loss: 1.6744e-01, Y_init: -2.3101e+00,   elapsed time: 315\n",
      "step:  1800,    loss: 1.6747e-01, Y_init: -2.3044e+00,   elapsed time: 319\n",
      "step:  1900,    loss: 1.6750e-01, Y_init: -2.3025e+00,   elapsed time: 323\n",
      "step:  2000,    loss: 1.6777e-01, Y_init: -2.2927e+00,   elapsed time: 327\n",
      "step:  2100,    loss: 1.6754e-01, Y_init: -2.3000e+00,   elapsed time: 331\n",
      "step:  2200,    loss: 1.6746e-01, Y_init: -2.3048e+00,   elapsed time: 335\n",
      "step:  2300,    loss: 1.6750e-01, Y_init: -2.3020e+00,   elapsed time: 339\n",
      "step:  2400,    loss: 1.6745e-01, Y_init: -2.3057e+00,   elapsed time: 344\n",
      "step:  2500,    loss: 1.6766e-01, Y_init: -2.2960e+00,   elapsed time: 348\n",
      "step:  2600,    loss: 1.6765e-01, Y_init: -2.2961e+00,   elapsed time: 352\n",
      "step:  2700,    loss: 1.6752e-01, Y_init: -2.3012e+00,   elapsed time: 356\n",
      "step:  2800,    loss: 1.6773e-01, Y_init: -2.2939e+00,   elapsed time: 360\n",
      "step:  2900,    loss: 1.6772e-01, Y_init: -2.2942e+00,   elapsed time: 365\n",
      "step:  3000,    loss: 1.6753e-01, Y_init: -2.3005e+00,   elapsed time: 369\n",
      "Done!!\n"
     ]
    }
   ],
   "source": [
    "losses,ys=[],[]\n",
    "start_time = time.time()\n",
    "\n",
    "# begin SGD iteration\n",
    "for step in range(3001):\n",
    "    x1_train,x2_train,y_train = bsde.sample(64)\n",
    "    model.train_on_batch([x1_train,x2_train], y_train)\n",
    "\n",
    "\n",
    "    y_pred = model.predict_on_batch([x1_valid, x2_valid])\n",
    "    loss = mean_squared_error(y_valid, y_pred)\n",
    "    y_init=model.get_layer('y_compute').get_weights()[0]\n",
    "    if step % 100 == 0:  # trace in terminal\n",
    "        elapsed_time = time.time() - start_time\n",
    "        print(\"step: %5u,    loss: %.4e, Y_init: %.4e,   elapsed time: %3u\" %\n",
    "              (step, loss, y_init, elapsed_time))\n",
    "    losses.append(round(loss,4))\n",
    "    ys.append(np.round(y_init, 4))\n",
    "print('Done!!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a701fae2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_scatter(x,y,x_name,y_name):\n",
    "    # 创建画图窗口\n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.set_title('{} with {}'.format(y_name,x_name))\n",
    "    ax.set_xlabel(x_name)\n",
    "    ax.set_ylabel(y_name)\n",
    "    ax.scatter(x, y,marker='.',s=1)\n",
    "    # plt.savefig(r'{} with {}.jpg'.format(y_name,x_name))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d044386d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAX7UlEQVR4nO3deZhldX3n8feHpgURGiRdaLOE1a3dEFvFoBgVcZ3gTFxHDUmMRB+d0YmOg8soGk00Og7kMTppVxIUk7gkuIIiCDEGLJC9ZW/D0tiF2HQroCzf+eOcsupWL1Q3de+tuuf9ep771LnnnnvP93dv96d+9Tu/e06qCklSd2w37AIkSYNl8EtSxxj8ktQxBr8kdYzBL0kdY/BLUscY/BqKJKuTHDHsOuZCkl8kOWALj49MWzUaDH7pPqqqnavqGoAkn03yvn7sJ0klOagfr61uMfglqWMMfg1dkh2SHJ/kxvZ2fJId2seWJvlaknVJbklydpLt2sf+V5IbkmxIcnmSZ7brt0tybJKrk/wsyT8m2b19bMckJ7Xr1yX5YZIHbaKmP0ry1Wn3r0ryj9PuX5fk4Ha5khyU5BjgFcBb2+Gfr057yYOTXJTk1iT/kGTHzbwXByX5XrvdzUn+oV1/VrvJhe1rv7Rd/4IkF7Rt+bckj5n2WquTvC3JZUl+nuQzk/vd0vuqDqgqb94GfgNWA0e0y+8F/h3YAxgD/g348/axvwT+H7C4vT0VCPAw4Dpgz3a7/YAD2+U3ta+3N7AD8LfAye1jfwp8FdgJWAQ8HliyifoOANbRdI6WAT8Bbpj22M+B7dr7BRzULn8WeN8m2nousCewO7AKeO1m3peTgXe0+90ReMq0x36zn/b+IcBa4EltW45u97XDtP1eAuzT7vf7k7Vt7n0d9r8Lb4O5+Rte88ErgPdW1dqqmgDeA7yqfexOmuDdt6rurKqzq6qAu2lCfXmSxVW1uqqubp/zp8A7qur6qvoVcBzwoiTbt6/3WzQBendVnVdV62cWVM2Y/QbgYOBpwKnADUke3t4/u6ru2Yo2/nVV3VhVt9D84jl4M9vdCexL8wvtjqr61y285muAv62qc9q2nAj8Cjh02jYfrarr2v2+H3j5tP1s6n1VBxj8mg/2pOlRT/pJuw7gQ8BVwGlJrklyLEBVXUXTsz8OWJvkC0kmn7Mv8JV2GGMdTQ/7buBBwN/ThPgX2mGlv0qyeDN1fQ/4XeDwdvlMmtB/Wnt/a9w0bfk2YOfNbPdWmr9ozk1yaZI/3sJr7gu8ebKdbVv3Yeq9g+avokn3+r6qGwx+zQc30oTYpN9u11FVG6rqzVV1APCfgD+bHMuvqs9X1VPa5xbwwfb51wHPrardpt12rKob2t7te6pqOfA7wAuAP9hMXZPB/9R2+Xvce/Dfp15zVd1UVa+pqj1p/nL52BZm8lwHvH9GO3eqqpOnbbPPtOVZva8afQa/5oOTgXcmGUuyFHgXcBL85uDlQUkCrKfpud+d5GFJntEeBL4DuL19DJqx6/cn2bd9jbEkR7XLT0/y6CSL2te7c9rzZvoe8HTg/lV1PXA28ByaoaIfbeY5P6U5BrBNkrw4yd7t3Z/T/CKZrG/ma38CeG2SJ6XxgCTPT7LLtG1en2Tv9uD224HJg8WbfF+3tW4tLAa/5oP3AePARcDFwPntOoCHAN8BfgH8APhYVZ1JM77/AeBmmmGUPWiCDeAE4BSaYYwNNAd6n9Q+9mDgizRht4om3E/aVFFVdUW737Pb++uBa4DvV9XmQvJTNMcd1iX55614DyY9ATgnyS/aNryxqq5tHzsOOLF97ZdU1TjNOP9HaX5JXAX84YzX+zxwWlv3Ndz7+6oOiMdzpNGUZDXwJ1X1nWHXovnFHr8kdYzBL0kd41CPJHWMPX5J6pjth13AbCxdurT222+/YZchSQvKeeedd3NVjc1cvyCCf7/99mN8fHzYZUjSgpLkJ5ta71CPJHWMwS9JHWPwS1LHGPyS1DEGvyR1jMEvSR1j8EtSx4x08FcVl954K56WQpKm9C34k3w6ydokl0xbt3uSbye5sv35wH7tH+CyNet53Unnc9majS6pKkmd1c8e/2dprlY03bHA6VX1EOD09n7fLF+2hI+/8hCWL1vSz91I0oLSt+CvqrOAW2asPgo4sV0+EXhhv/YPkIRH7rkrzdXlJEkw+DH+B1XVGoD25x6b2zDJMUnGk4xPTEwMrEBJGnXz9uBuVa2sqhVVtWJsbKOTy0mSttGgg/+nSZYBtD/XDnj/ktR5gw7+U4Cj2+WjgX8Z8P4lqfP6OZ3zZOAHwMOSXJ/k1cAHgGcluRJ4VntfkjRAfbsQS1W9fDMPPbNf+5Qk3bt5e3B3LvjNXUna2EgHv9/claSNjXTw+81dSdrYgrjY+raa/OauJGnKSPf4JUkbM/glqWMMfknqGINfkjpmpIPfefyStLGRDn7n8UvSxkY6+J3HL0kbcx6/JHXMSPf4JUkbM/glqWMMfknqGINfkjrG4Jekjhnp4PcLXJK0sZEOfr/AJUkbG+ng9wtckrQxv8AlSR0z0j1+SdLGDH5J6hiDX5I6xuCXpI4Z+eB3Lr8k9Rr54HcuvyT1Gvngdy6/JPUa6Xn84Fx+SZpp5Hv8kqReQwn+JP8jyaVJLklycpIdh1GHJHXRwIM/yV7AfwdWVNWjgEXAywZdhyR11bCGerYH7p9ke2An4MYh1SFJnTPw4K+qG4APA/8BrAFurarT+rg/5/FL0jTDGOp5IHAUsD+wJ/CAJK/cxHbHJBlPMj4xMbHN+3MevyT1GsZQzxHAtVU1UVV3Al8GfmfmRlW1sqpWVNWKsbGxbd6Z8/glqdcwgv8/gEOT7JQkwDOBVf3a2eQ8/mZXkqRhjPGfA3wROB+4uK1h5aDrkKSuGso3d6vq3cC7h7FvSeo6v7krSR0z8sHvdE5J6jXywe90TknqNfLB73ROSerlaZklqWNGvscvSepl8EtSxxj8ktQxIx/8TueUpF4jH/xO55SkXiMf/E7nlKReTueUpI4Z+R6/JKmXwS9JHWPwS1LHjHzwO51TknqNfPA7nVOSeo188DudU5J6OZ1Tkjpm5Hv8kqReBr8kdczIB7+zeiSp18gHv7N6JKnXyAe/s3okqZezeiSpY0a+xy9J6mXwS1LHjHzwO6tHknqNfPA7q0eSeo188DurR5J6OatHkjpmKD3+JLsl+WKSHydZleTJw6hDkrpoWD3+E4BvVdWLktwP2GlIdUhS5wy8x59kCXA48CmAqvp1Va3r1/6c1SNJvYYx1HMAMAF8JsmPknwyyQNmbpTkmCTjScYnJia2eWfO6pGkXsMI/u2BQ4CPV9XjgF8Cx87cqKpWVtWKqloxNja2zTtzVo8k9RpG8F8PXF9V57T3v0jzi6AvJmf1JOnXLiRpQRl48FfVTcB1SR7WrnomcNmg65Ckrtrq4E/ywCSPuY/7/W/A55JcBBwM/MV9fL3N8uCuJPWaVfAnOTPJkiS7AxfSHJj9yLbutKouaMfvH1NVL6yqn2/ra90bD+5KUq/Z9vh3rar1wH8BPlNVjweO6F9Zc8eDu5LUa7bBv32SZcBLgK/1sZ4558FdSeo12+B/L3AqcHVV/TDJAcCV/StLktQvswr+qvqndjz+de39a6rq9/tb2tzw4K4k9Zrtwd2HJjk9ySXt/cckeWd/S5sbHtyVpF6zHer5BPA24E6AqroIeFm/ippLHtyVpF6zDf6dqurcGevumuti+sGDu5LUa7bBf3OSA4ECSPIiYE3fqpIk9c1sz8f/emAl8PAkNwDXAq/sW1VzrKq4bM16li9bYs9fUufNdlbPNVV1BDAGPLyqnlJVq/ta2RzyAK8kTZntrJ43thdQuQ34v0nOT3Jkf0ubOx7glaQpsx3j/+P2lA1HAnsAfwR8oG9VzTEP8ErSlNkG/2RiPo/mXD0XTlsnSVpAZhv85yU5jSb4T02yC3BP/8qaW357V5KmzDb4X01zecQnVNVtwGKa4Z4FwYO7kjRltsH/ZODyqlqX5JXAO4Fb+1fW3PLgriRNmW3wfxy4LcljgbcCPwH+rm9VzTEP7krSlNkG/13VDJAfBZxQVScAu/SvrLnlGL8kTZlt8G9I8jbgVcDXkyyiGedfEBzjl6Qpsw3+lwK/opnPfxOwF/ChvlU1xxzjl6Qpsz1lw03A54Bdk7wAuKOqHOOXpAVotqdseAlwLvBimuvuntOeoVOStMDM9uyc76CZw78WIMkY8B3gi/0qbC55dk5JmjLbMf7tJkO/9bOteO7QeXBXkqbMtsf/rSSnAie3918KfKM/Jc09D+5K0pRZBX9V/c8kvw8cRnNytpVV9ZW+VjaHJg/uSpJm3+Onqr4EfKmPtfSNY/ySNGWL4/RJNiRZv4nbhiQLZsDcMX5JmrLFHn9VLZjTMmyJY/ySNGXBzMyRJM2NTgS/Qz2SNGVowZ9kUZIfJflav/flUI8kTRlmj/+NwKpB7Mhz9UjSlKEEf5K9gecDnxzE/jwfvyRNGVaP/3iaK3lt9oLtSY5JMp5kfGJi4j7tzDF+SZoy8OBvT+u8tqrO29J2VbWyqlZU1YqxsbH7tE/H+CVpyjB6/IcBv5dkNfAF4BlJTurnDh3jl6QpAw/+qnpbVe1dVfsBLwO+W1Wv7PM+HeOXpJbz+CWpY2Z9krZ+qKozgTP7vR/H+CVpSid6/JKkKZ0Ifod6JGlKJ4LfoR5JmtKJ4E/C8mVLuGzNemf2SOq8TgQ/ONwjSZM6E/wO90hSozPBL0lqdCb4HeqRpEZngt+hHklqdCb4JUmNzgS/Qz2S1OhM8DvUI0mNzgS/JKnRmeB3qEeSGp0Jfod6JKnRmeCXJDU6E/wO9UhSozPB/4gH78Jbnv1QHvHgXYZdiiQNVWeCf9VNG/jwqVew6qYNwy5FkoaqM8HvwV1JanQm+CVJjc4Evwd3JanRmeD34K4kNToT/B7claRGZ4LfHr8kNToT/Pb4JanRmeC3xy9Jjc4Evz1+SWp0JviXL1vCx17xOCioqmGXI0lD05ngT0ISXvc55/JL6raBB3+SfZKckWRVkkuTvHFQ+3acX5KG0+O/C3hzVT0COBR4fZLlg9ix4/ySNITgr6o1VXV+u7wBWAXsNYh92+OXpCGP8SfZD3gccM4g9mePX5KGGPxJdga+BLypqjY62prkmCTjScYnJibmZJ/2+CVpSMGfZDFN6H+uqr68qW2qamVVraiqFWNjY3Oy38vWrOcvvv5jZ/VI6rRhzOoJ8ClgVVV9ZKD7Jj0/JamLhtHjPwx4FfCMJBe0t+cNYsePWLYLb3/+w3nEMod6JHXX9oPeYVX9Kwynyz051HPA0gfwqL12G0YJkjR0nfnmLjjUI0nQseB3qEeSOhb8zuqRpI4Fv0M9ktSx4HeoR5I6FvyrbtrAh751OV+7eI3n5JfUWZ0K/uXLlvCWZz+U9399FZfeeOuwy5GkoehU8CfhwKU7c9fdBXb4JXVUp4JfkmTwS1LnGPyS1DEGvyR1jMEvSR1j8EtSxxj8ktQxnQv+JCwKnHXlBPfcc8+wy5Gkgetc8C/fcwlHH7YvHz7tCr560Y3DLkeSBq5zwZ+Epx64lJ0WL2L/3XcadjmSNHCdC36Aa27+Jb/89d1cc/Mvh12KJA1cJ4OfquZUPZ6hU1IHdTL4k+ZCLDesu8PTM0vqnE4G/4F77MIu99uOlWdfyyU3rBt2OZI0UJ0M/kfutSvHHL4/t95xF1f9dMOwy5Gkgepk8Cf5zfn4r7/ltuEWI0kD1sngB0iapp+7eh133333kKuRpMHpbPA/7WFjLALOvvpn/M0ZVwy7HEkamM4G/6P23o0jl48BcPqqCXv9kjqjs8GfhEfutRsAF96wgf/9zxc6tVNSJ3Q2+AFe97sH8tu77wjA53+4hhO+vcoTt0kaeZ0O/kWLFvGdNz2VHbZv7h//3Wt5+Nu/yeEfPJ3bb799uMVJUp9kIQxvrFixosbHx/v2+rfffjuPfu93uWvGW7Hr9rB4Efz6ro2fk/iYj/nYfHtsvtQxV4/d//6L+dh/fSyH7L/Hb844sDWSnFdVKzZaP4zgT/Ic4ARgEfDJqvrAlrbvd/AD3HHHHRz5kTO45Zf38AuP80qaJxZtB6e84Sk8cs9dt/q5mwv+7eeksq0rZBHwN8CzgOuBHyY5paouG3Qt0+24446c9fbnUlWMX30Tf3Li+VDzsxfgYz7mY93q8S9ftmTjDe+DgQc/8ETgqqq6BiDJF4CjgKEG/6QkPOGgZVz4588fdimS1BfDOLi7F3DdtPvXt+t6JDkmyXiS8YmJiYEVJ0mjbhjBv6kjFBsdaKiqlVW1oqpWjI2NDaAsSeqGYQT/9cA+0+7vDXjxW0kakGEE/w+BhyTZP8n9gJcBpwyhDknqpIEf3K2qu5K8ATiVZjrnp6vq0kHXIUldNYxZPVTVN4BvDGPfktR1nT5lgyR10YI4ZUOSCeAn2/j0pcDNc1jOMNmW+WdU2gG2Zb66L23Zt6o2mha5IIL/vkgyvqmvLC9EtmX+GZV2gG2Zr/rRFod6JKljDH5J6pguBP/KYRcwh2zL/DMq7QDbMl/NeVtGfoxfktSrCz1+SdI0Br8kdcxIB3+S5yS5PMlVSY4ddj33JsnqJBcnuSDJeLtu9yTfTnJl+/OB07Z/W9u2y5M8e3iVQ5JPJ1mb5JJp67a69iSPb9+Dq5L8dbblenP9actxSW5oP5sLkjxvvrclyT5JzkiyKsmlSd7Yrl9wn8sW2rIQP5cdk5yb5MK2Le9p1w/uc6mqkbzRnAfoauAA4H7AhcDyYdd1LzWvBpbOWPdXwLHt8rHAB9vl5W2bdgD2b9u6aIi1Hw4cAlxyX2oHzgWeTHP67m8Cz50nbTkOeMsmtp23bQGWAYe0y7sAV7T1LrjPZQttWYifS4Cd2+XFwDnAoYP8XEa5x/+bK31V1a+BySt9LTRHASe2yycCL5y2/gtV9auquha4iqbNQ1FVZwG3zFi9VbUnWQYsqaofVPOv+u+mPWdgNtOWzZm3bamqNVV1fru8AVhFc9GjBfe5bKEtmzOf21JV9Yv27uL2Vgzwcxnl4J/Vlb7mmQJOS3JekmPadQ+qqjXQ/OMH9mjXL4T2bW3te7XLM9fPF29IclE7FDT5Z/iCaEuS/YDH0fQuF/TnMqMtsAA/lySLklwArAW+XVUD/VxGOfhndaWveeawqjoEeC7w+iSHb2Hbhdi+SZurfT636ePAgcDBwBrg/7Tr531bkuwMfAl4U1Wt39Kmm1g339uyID+Xqrq7qg6muRDVE5M8agubz3lbRjn4F9yVvqrqxvbnWuArNEM3P23/pKP9ubbdfCG0b2trv75dnrl+6Krqp+1/1nuATzA1rDav25JkMU1Qfq6qvtyuXpCfy6baslA/l0lVtQ44E3gOA/xcRjn4F9SVvpI8IMkuk8vAkcAlNDUf3W52NPAv7fIpwMuS7JBkf+AhNAd65pOtqr3983ZDkkPb2Ql/MO05QzX5H7L1n2k+G5jHbWn3+ylgVVV9ZNpDC+5z2VxbFujnMpZkt3b5/sARwI8Z5OcyyKPZg74Bz6M5+n818I5h13MvtR5Ac+T+QuDSyXqB3wJOB65sf+4+7TnvaNt2OUOY/TKj/pNp/tS+k6Yn8uptqR1YQfOf92rgo7TfLp8Hbfl74GLgovY/4rL53hbgKTR/+l8EXNDenrcQP5cttGUhfi6PAX7U1nwJ8K52/cA+F0/ZIEkdM8pDPZKkTTD4JaljDH5J6hiDX5I6xuCXpI4x+KV7keRNSXYadh3SXHE6p3QvkqwGVlTVzcOuRZoL9viladpvUH+9PVf6JUneDewJnJHkjHabI5P8IMn5Sf6pPX/M5PUUPtiea/3cJAe161/cvtaFSc4aXuukhsEv9XoOcGNVPbaqHgUcT3P+k6dX1dOTLAXeCRxRzQn1xoE/m/b89VX1RJpvUR7frnsX8Oyqeizwe4NphrR5Br/U62LgiLbn/tSqunXG44fSXBjj++1pdY8G9p32+MnTfj65Xf4+8Nkkr6G5QJA0VNsPuwBpPqmqK5I8nuY8MH+Z5LQZm4Tm/Okv39xLzFyuqtcmeRLwfOCCJAdX1c/munZptuzxS9Mk2RO4rapOAj5McwnGDTSX+wP4d+CwaeP3OyV56LSXeOm0nz9otzmwqs6pqncBN9N7il1p4OzxS70eDXwoyT00Z+d8Hc2QzTeTrGnH+f8QODnJDu1z3klzFliAHZKcQ9Opmvyr4ENJHkLz18LpNGdglYbG6ZzSHHHapxYKh3okqWPs8UtSx9jjl6SOMfglqWMMfknqGINfkjrG4Jekjvn/p4InqLGThU4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_scatter(list(range(3001)), losses, 'steps', 'losses')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "48a2031a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAhzUlEQVR4nO3deZwcdZ3/8dd7rhzkAjJADiTcJuFmRFhWvBACghFWBFZW3NUfPxB+rnu4i7KL+HjsriIe6C6Li7IueCG7irCCcokccmWChFwEYkggZEImHJmEJHN+fn9UDdPT0zPdk5memuP9fDz60dVV1VWfb1dPf+Z7VJUiAjMzs75UZB2AmZkNf04WZmZWlJOFmZkV5WRhZmZFOVmYmVlRThZmZlaUk4WNWpJ+JenCwV53MEj6gqTv9bH8E5IeGap4zIqRz7Mw607SJ4BPRcQfD9H+5gAvANUR0TaYMUi6CjgoIi4YYJg2xrlmYWZmRTlZ2LAn6XOSfpY3718lXVvkfb+V9Kl0+hOSHpH0NUmvS3pB0mn560qaC3wHOEHSNklvpMtPl7RC0lZJL0v62172uU7Ssen0BZJC0rz09ack/SKdvkrSD9O3PZQ+v5Hu84Sc7RWMt8B+/z6Na6ukVZLeL2kB8AXg3HS7S9J1p0q6UVJD+p5/klSZ8zn9Lv18t0h6VtL7c/bzCUlr0v28IOljfR0DGz2cLGwk+CGwQNI0AElVwLnAD/q5nXcCq4DpwFeBGyUpd4WIWAlcDDwWEZMiYlq66Ebg/0bEZOAw4De97ONB4D3p9EnAGuDdOa8fLPCek9Lnaek+Hys1XgBJhwKXAe9I4zsVWBsRvwb+Bfhput0j07fcBLQBBwFHA6cAn8rZ5DvTuKcDXwR+LmkPSbsB3wZOS/fzR8DTvXwONso4WdiwFxENJP99n5POWgBsjojF/dzUuoj4bkS0k/xgzgD2LvG9rcA8SVMi4vWIeKqX9R6kKzm8C/hyzut3UzhZDDTedmBcGl91RKyNiD8U2qCkvYHTgM9GxJsRsQn4JnBezmqbgGsjojUifkqSsD6YLusADpM0ISIaImJ5P8pjI5iThY0UNwGdnbQX0P9aBcDGzomI2J5OTirxvX8CnA6sk/RgblNRngeBd0naB6gEfgqcmHZiT6V//4mXFG9ErAY+C1wFbJJ0i6SZvWxzP6AaaJD0RtrM9h/AXjnrvBzdR76sA2ZGxJskNbqL0/ffKent/SiPjWBOFjZS/AI4QtJhwBnAj8q4rx5DBCNiUUQsJPlR/QVwa8E3Jj/c24HPAA9FxFaSH/2LgEcioqOU/fU74IgfpyOn9ku3d3Uv234JaAamR8S09DElIubnrDMrr7nrbcCGdD93R8QHSGo5zwLfHWjsNjI4WdiIEBE7gf8Bfgw8GREvlnF3rwCzJdUASKqR9DFJUyOiFWgiafrpzYMkfQidTU6/zXudr5GkeeeAXQlW0qGS3idpHLAT2JET3yvAHEkV8FaT3j3A1yVNkVQh6UBJ787Z5F7AZyRVSzoHmAvcJWlvSR9K+y6agW30/TnYKOJkYSPJTcDh7FoTVH/8BlgObJS0OZ33Z8BaSU0kzTB9nbfwIDCZrlFO+a+7SZuY/hn4Xdo0dHw/4x0HfAXYTFKL2YtkFBTAf6fPr0rq7Gf5OFADrABeJ0nCM3K29wRwcLq9fwY+EhGvkvxe/A1JLeM1kj6YT/czVhuhfFKejRiS3kbS9LFPRDRlHc9oNNQnJNrI4ZqFjQhpM8pfA7c4UZgNvaqsAzArJm0jf4VkVM6CvGXbennbaRHxcLljMxsr3AxlZmZFuRnKzMyKGpXNUNOnT485c+ZkHYaZ2YiyePHizRFRW2jZqEwWc+bMob6+PuswzMxGFEnrelvmZigzMyvKycLMzIpysjAzs6KcLMzMrCgnCzMzK8rJwszMinKyMDOzopws8kQEyzdswZdBMTPr4mSRZ0VDE5f88ClWNPjCpmZmnZws8sybMYXrLziGeTOmZB2KmdmwMSov9zEQkpg/c2rWYZiZDSuZ1ywkLZC0StJqSZcXWP4eSVskPZ0+rswiTjOzsSzTmoWkSuA64APAemCRpDsiYkXeqg9HxBlDEVNEsKKhiXkzpiBpKHZpZjbsZV2zOA5YHRFrIqIFuAVYmGVA7uA2M+sp62QxC3gp5/X6dF6+EyQtkfQrSfMLbUjSRZLqJdU3NjbuckDu4DYz6ynrZFGonSf/BIengP0i4kjgX4FfFNpQRNwQEXURUVdbW/DeHaUFlHZwuwnKzKxL1sliPbBvzuvZwIbcFSKiKSK2pdN3AdWSpg9diGZmlnWyWAQcLGl/STXAecAduStI2kfpv/mSjiOJ+dVyBeQzuM3Meso0WUREG3AZcDewErg1IpZLuljSxelqHwGWSVoCfBs4L8r4S+4ObjOznjQa/4Ouq6uLXb0Ht4fOmtlYJWlxRNQVWuYzuPP4DG4zs56y7rMYdtxnYWbWk5NFHvdZmJn15GSRxyflmZn15D6LPO6zMDPryTWLPO6zMDPryckij/sszMx6crLI4z4LM7Oe3GeRx30WZmY9uWZhZmZFOVnkcQe3mVlPThZ53MFtZtaTk0Ued3CbmfXkDu487uA2M+vJNYs87rMwM+vJySKP+yzMzHpyssjjPgszs57cZ5HHfRZmZj25ZpHHfRZmZj05WeRxn4WZWU9OFnncZ2Fm1pP7LPK4z8LMrCfXLApwv4WZWXdOFgW438LMrDsniwLcb2Fm1l3myULSAkmrJK2WdHmB5ZL07XT5M5KOGYKYmD9zKpLKvSszsxEh02QhqRK4DjgNmAecL2le3mqnAQenj4uA68sdl/sszMy6y7pmcRywOiLWREQLcAuwMG+dhcDNkXgcmCZpRjmDcp+FmVl3WSeLWcBLOa/Xp/P6uw6SLpJUL6m+sbFxQEG5z8LMrLusk0WhToH8tp9S1iEiboiIuoioq62tHZTgzMwskXWyWA/sm/N6NrBhF9YZVG6GMjPrLutksQg4WNL+kmqA84A78ta5A/h4OirqeGBLRDSUMyg3Q5mZdZfp5T4iok3SZcDdQCXwnxGxXNLF6fLvAHcBpwOrge3An5c7Ll/yw8ysu8yvDRURd5EkhNx538mZDuDSIY6JFQ1NzJsxxedamJmRfTPUsOQ+CzOz7pwsCnCfhZlZd5k3Qw1H7rMwM+vONYsCfLkPM7PunCwKcJ+FmVl3ThYFuM/CzKw7JwszMyvKyaIAN0OZmXXnZFGAm6HMzLrz0NkCPHTWzKw71ywK8NBZM7PunCwKcJ+FmVl3ThYFuM/CzKw7JwszMyvKyaIAN0OZmXXnZFGAm6HMzLpzsjAzs6KcLApwM5SZWXdOFgW4GcrMrDsnCzMzK8rJogA3Q5mZdedkUYCboczMunOyKEAS82ZMYUVDk68PZWaGk0Wv3BRlZtbFyaIXbooyM+uS2f0sJO0B/BSYA6wFPhoRrxdYby2wFWgH2iKibuiiNDMzyLZmcTlwf0QcDNyfvu7NeyPiqKFMFG6GMjPrkmWyWAjclE7fBHw4u1B6cjOUmVmXLJPF3hHRAJA+79XLegHcI2mxpIt625ikiyTVS6pvbGwsQ7hmZmNXWZOFpPskLSvwWNiPzZwYEccApwGXSjqp0EoRcUNE1EVEXW1t7YBjdzOUmVmXsnZwR8TJvS2T9IqkGRHRIGkGsKmXbWxInzdJug04DnioLAHncDOUmVmXLJuh7gAuTKcvBG7PX0HSbpImd04DpwDLhixCMzMDsk0WXwE+IOl54APpayTNlHRXus7ewCOSlgBPAndGxK+HIjg3Q5mZddFovJxFXV1d1NfXD2gbEcGKhibmzZiCpEGKzMxs+JK0uLdTFHwGt5mZFeVk0Qs3Q5mZdXGy6IVHQ5mZdXGyMDOzopwseuFmKDOzLk4WvZi7z2T+9tRDmLvP5KxDMTPLXEnJQtKJ6UlxSLpA0jck7Vfe0LK1cuNWvnb3c6zcuDXrUMzMMldqzeJ6YLukI4G/A9YBN5ctqmHAHdxmZl1KTRZtkZy9txD4VkR8C3D7jJnZGFFqstgq6fPABcCdkiqB6vKFlT13cJuZdSk1WZwLNAOfjIiNwCzgmrJFNQy4g9vMrEupyeKvIuIbEfEwQES8CMwvX1jZcwe3mVmXUpPFBwrMO20wAxlu3MFtZtalz5sfSboE+DRwgKRnchZNBh4tZ2BmZjZ8FKtZ/Bg4k+RGRWfmPI6NiI+VObZMuYPbzKxLsduqRkSslXRp/gJJe0TEa2WKK3NuhjIz61IsWfwYOANYDASQexegAA4oU1yZk8S8GVN8AyQzM4o0Q0XEGenz/hFxQPrc+Ri1iaKTm6LMzBLFahZvkTQL2C/3PRHxUDmCGi58roWZWaKkZCHpapIT81YA7ensAEZ1sug81+LA2knMnzk163DMzDJTas3iw8ChEdFcxliGHdcszMwSpZ6Ut4ZRfi2oQnwWt5lZotSaxXbgaUn3k1wjCoCI+ExZohomPHzWzCxRarK4I32YmdkYVFKyiIibyh3IcNQ5dPb6C45xB7eZjWl99llIujV9XirpmfzHQHYs6RxJyyV1SKrrY70FklZJWi3p8oHss7/cwW1mlihWs/jL9PmMMux7GXA28B+9rZDeZOk6kqvergcWSbojIlaUIZ4ePHTWzCzRZ7KIiIb0eV1f60l6LCJO6M+OI2Jl+t6+VjsOWB0Ra9J1byG5teuQJAvXLMzMEqUOnS1m/CBtJ98s4KWc1+vTeT1IukhSvaT6xsbGQdm5h86amSVKvtxHEVFopqT7gH0KLLoiIm4vYbuFqh0F9xURNwA3ANTV1RVcp79cszAzSwxWsigoIk4e4CbWA/vmvJ4NbBjgNkvmPgszs0RJzVCSLpO0e1+rDFI8+RYBB0vaX1INcB5DeL6HaxZmZolS+yz2IRmJdGs6lDU/OfxZf3cs6SxJ64ETgDsl3Z3OnynpLoCIaAMuA+4GVgK3RsTy/u5rV7nPwswsoYjSmvfTBHEK8OdAHXArcGNE/KF84e2aurq6qK+vH/B2Ojo6+OXSBs44fAYVFYM1FsDMbHiStDgiCp73VvIvYCRZZWP6aAN2B/5H0lcHJcphyDULM7NEqfez+AxwIbAZ+B7wuYholVQBPA/8XflCzI77LMzMEqWOhpoOnJ1/cl5EdEgqx9ndw4JHQ5mZJUq9kOCVfSxbOXjhDC/zZkzh3z92NARERLGzzc3MRi332vZBEpK45EdPsaKhKetwzMwy42RRhPstzMycLIryiCgzMyeLolyzMDNzsijKNQszMyeLolyzMDNzsihqRUMT/3Lnsx4NZWZjmpNFEUovqKuyXVjXzGz4c7IoYu6MyXzhg29n7gw3Q5nZ2OVkUYQ7uM3MnCyKcge3mZmTRVGuWZiZOVkU5ZqFmZmTRVEeOmtm5mRRlIfOmpk5WRTlobNmZk4WRa3cuJVrfr2KXy5tILkNuZnZ2ONkUcS8GVP43IJDuebuVe63MLMxq9R7cI9Zkjjj8BkAHhFlZmOWaxYl8IgoMxvrnCxK4BFRZjbWZZYsJJ0jabmkDkl1fay3VtJSSU9Lqh/KGDt5RJSZjXVZ1iyWAWcDD5Ww7nsj4qiI6DWplJObocxsrMusgzsiVkLSgTzsBbS2d4BHzprZGDUS+iwCuEfSYkkX9baSpIsk1Uuqb2xsHMLwzMxGv7LWLCTdB+xTYNEVEXF7iZs5MSI2SNoLuFfSsxHRo+kqIm4AbgCoq6sb1DqAJKorK0ZGLcjMrAzKmiwi4uRB2MaG9HmTpNuA4yitn2PQuIPbzMa6Yd0MJWk3SZM7p4FTSDrGh5Q7uM1srMty6OxZktYDJwB3Sro7nT9T0l3pansDj0haAjwJ3BkRvx7yYN3BbWZjXJajoW4DbiswfwNwejq9BjhyiEPrwX0WZjbWDetmqOFi3swpfO/CYwF85VkzG5OcLErQWaP45E31LN+wJeNozMyGnpNFqdxvYWZjmJOFmZkV5WRhZmZFOVmYmVlRThZmZlaUk4WZmRXlZGFmZkU5WfRDRLCmcZtPzDOzMcfJokSdJ+Z96ZcrfWKemY05ThYlmjdzCl88c27ywhULMxtjnCxKJIkDa30/CzMbm5wszMysKCcLMzMrysmiHyKCtvYOj4YyszHHyaIf1mx+k60721iz+c2sQzEzG1JOFv1wYO0kpk6o4uXXd9DR0ZF1OGZmQ8bJoh/mz5rKp07an6/d+xz/+8yGrMMxMxsyThb9IImTDqpl8rgqDthzt6zDMTMbMk4WZmZWlJNFP7mT28zGIieLftp/z4lMrKlk/z0nZh2KmdmQcbLopxde3c72lnZeeHV71qGYmQ0ZJ4t+8vBZMxuLMksWkq6R9KykZyTdJmlaL+stkLRK0mpJlw9xmD3MnzWVT75rDtfc8xx3LHk563DMzIZEljWLe4HDIuII4Dng8/krSKoErgNOA+YB50uaN6RR9oyJ2VMnJFcp7/BlP8xsbMgsWUTEPRHRlr58HJhdYLXjgNURsSYiWoBbgIVDFaOZmSWGS5/FXwC/KjB/FvBSzuv16bweJF0kqV5SfWNjYxlCNDMbu8qaLCTdJ2lZgcfCnHWuANqAHxXaRIF5Bdt+IuKGiKiLiLra2trBKUAvKiqSj+33L21xJ7eZjQlV5dx4RJzc13JJFwJnAO+Pwtf9Xg/sm/N6NpD5RZnOOHImi9a9ys2Pv8jR+03jw0fvW/xNZmYjWJajoRYAfw98KCJ6O2lhEXCwpP0l1QDnAXcMVYy9qaio4JjZ0wgg2l2zMLPRL8s+i38DJgP3Snpa0ncAJM2UdBdA2gF+GXA3sBK4NSKWZxVwrpff2Nnt2cxsNCtrM1RfIuKgXuZvAE7PeX0XcNdQxVWq2bsnl/vYtK2Fjo6Ot/oxzMxGI//C7aKD9p7MhEq4+fEXuf3p9VmHY2ZWVk4Wu2j+rKmcfUwyine9rxNlZqOck8UuksQ+UyYA8OzGbR5Ca2ajmpPFALwtvVvenctfcVOUmY1qThYDcOZRMzn17XsC8GLjtoyjMTMrHyeLAaioqKC6uhqAWxZvoLW1NeOIzMzKw8ligL750SOYPrGChqZmLvz+ExQ+Ed3MbGRzshig6upqjj8wuRbVo2u28NQLmzKOyMxs8DlZDIJvfvRIJo1LPsoLvv8ULS0tGUdkZja4nCwGQXV1NfWffy/Vgh2tHRzzT/fT3NycdVhmZoPGyWKQjB8/nvPemVx9dltLB8d9+QF3eJvZqOFkMYi+eMY8jpiZnHuxZWc7R3zpXnbs2JFxVGZmA+dkMYiqqqq47dJ3cUhtcpHBHW3B3C/9hlO+/gA7d/rqtGY2cmk0DvWsq6uL+vr6zPbf2trKBd99lCde7H6i3unza/n6Rw7nytuf4Z5lmxk/vpp//9MjOfJte3LtPcv5waPrqamp5Owjarlt6WbOPXYmbzQH59TN5pDaiZz974/QtL2Zph0gQXUltKR3Me98rcpKLjh+Py57/yFUVlbS2trKX/709xy61yTaWtv59bObueSk/TnzqFn8x8NrmD11IgftPZn5s6Yidb8xYUtLCxd+/wn2mDSBaz965FvnlPRXRLCioYl5M6YgiZ07d3LqN3/Lmzvb34q/pqaCUw+bwZcWHkZV1a5fDLmjo4NfLm3gjMNn9LgScHNzM+ffuIiffPId1NTU8Pu1jfzjL5YSEYyvhDWvNlM7ZQJXn30YEcHnfvYMM6ftxn9+oo6ampqSY2hra+MLP3uKB1Y20h5wYO1utHe0s3rTTojkWE2aWM05x85izvQpPT7/7du3c8KXH6CqUhx34HS+fd7RtLe3c/q3H2aP8ZVs3LqT5vaKbse52Gfel5aWFv78pkUcf8B0Pv2eg6isrGT79u380VceYOK4Kt43d2+u+lByXCKC369t5IqfL6GxqYWKqir+5uQDadjSzOL1W7nx48dQU1PDjh07ePdXf0N7dJUhIrjge4+ycctOKhB7TKhkzWvN7DV1IndcegLjx48nIli+YQs7d+zk0z9azM50rEjn97uiqoq/O/UQzq7bb5eu9BwRLF3/Og+u2kRrSxv3rGhg89YWWtqSfUwcX8nhs6YwffJ4qqqqufKMuVRUVPCz+rXc/Og63jZ9Eteee1SPv4XOz/uA3WtYcO2DtFPJpSftx53LN3PavD351n2raQuYO2t3vp9+n5qbm1n4rw+yaWtrt/mFbN++nXd//SFOmbcPV31oPisamvjh71Zz74rNVFUkvwPjxlX2+Z0ohaTFEVFXcJmTRXlEBPV/2MiF//UU29v6Xnd8BewscmmpCqA/V5+qBiZUwvYOaCvhEE8Aaqq6J6D8904p8P2TYFw11FRA086ey6orYXsLbI9kH9WVsLW9l3vjpqZWwYRxFRy091S++2dHc+NjL7Jnjbj67ueproSIrhjz99cObGvrKn/usq1tXZ/hpErY1l7sUyle9tzPq9POgJZ+Xiqs87PJj7NTb8e/gqQs+bG0dXR95vnHNb8M29uhNbpvr9AxmlIJ7QFvFinb1KrCZahS39/FqVVQUQGvlzCYMPf49nYcCi1rB97sx3EHmFABO/IKM31c9/11fi6i7+92pymVSRz5ofT2PSv0efbmbbuP476/Oqlf/+B07cvJIjMdHR3895Nr+OL/rmJn3jej0Berkp5foHzVwMS8H4DOL2t/k0qWOn8gofSkVk6l/qH3R7W6fogh+TGQkh+KYuXNf2+ucsRaSE1F74mvGmgrcxy5SbS6El5rLs/3ewJQUdn/RNIfNYKWAh/WQI9lZ+LKTT4nHbInN//F8f3eVl/JIrObH40VFRUVnHv8QZx97H5cfPPjPP1SE2+fmVQ529vbec81vwHEjN0n8dOL3gnA+Tcu4r8uOIKPfX8xtRMrWfryVqio5E+P25cD9prKmUfN7FEF76wGH7jHOM667mEam5KqdXW1OHL2ZDY2NVNVVcX5x87kpkfX8srWVvaaXM32ljZ2tMRb1fDcBFRTU8H7Dp3Oq9uaeWb9FpoLDO6Skj/erb38N19dCa3t3bc7fnzPpoSWlhb+9IZHeG7jDkTyR7Wjlz/cQv99de6vr1rOpInVnDl/T36yaCOQvP7IMTOprKiisrKSi06aw88Xr+Mbd6+iuqaG//ee/Vm89jUeWNXYa9kL/UdbVdXV3FBRUcEX71hKZUUVV54x960mtubmZs667mG27mh56/Pv3ObUyeO5+zN/RFVVFRff/Dj1LzQxblzS3NP4Zjv77rEbp8zdk7Oue5gNr7cWjCX3dbH/vHebUMWJB0zjmfVbaHgj2d7EiTX85q9OpKqq6q3jktuEdu479uXS9x1Ca2srp33rIXarqeLQvSZw38rXkmM0aRzfOvcIDps1jUt+8AT1LzRRXV3BKfP34aPv2Jc1jW+yZH0TV5x+KD9fvI4v3/kckNSCAMaPT95/9Jzabs1oO3fu5JRvPMAbOdWb/tQsJBhfI/aYWE11TTUXnjCHQ2dMY/6sqQAsefFVrn/gebbsaGVCFTy1bisS7Dl5HB88bC+WvbyFp15seqvpJ38/ldXVXHf+EQjx97ct4+iZu7Fk406uPms+h++7+1vNzRFdzUYXv/sA/vEXS3hgZe/fs6mTx/O/F9fxyR8s5vlXdjChJql55zaRNjc38yfX/45pu43nexcc03NDA+SahQ1YX/0Eu6qtrY3P/89i7l+5mdpJ1URlNbdfcjxrXm8pqR3ezPrPzVBmZlZUX8nCQ2fNzKwoJwszMyvKycLMzIpysjAzs6KcLMzMrCgnCzMzK8rJwszMihqV51lIagTWDWAT04HNgxROlkZLOcBlGa5GS1lGSzlgYGXZLyJqCy0YlclioCTV93ZiykgyWsoBLstwNVrKMlrKAeUri5uhzMysKCcLMzMrysmisBuyDmCQjJZygMsyXI2WsoyWckCZyuI+CzMzK8o1CzMzK8rJwszMinKyyCFpgaRVklZLujzreEohaa2kpZKellSfzttD0r2Snk+fd89Z//Np+VZJOjW7yEHSf0raJGlZzrx+xy7p2PQzWC3p2xriOyP1Uo6rJL2cHpenJZ0+3MuRxrCvpAckrZS0XNJfpvNH1HHpoxwj7rhIGi/pSUlL0rJ8KZ0/tMckIvxI+m0qgT8ABwA1wBJgXtZxlRD3WmB63ryvApen05cDV6fT89JyjQP2T8tbmWHsJwHHAMsGEjvwJHACye2MfwWcNgzKcRXwtwXWHbblSGOYARyTTk8GnktjHlHHpY9yjLjjku53UjpdDTwBHD/Ux8Q1iy7HAasjYk1EtAC3AAszjmlXLQRuSqdvAj6cM/+WiGiOiBeA1STlzkREPAS8lje7X7FLmgFMiYjHIvlruDnnPUOil3L0ZtiWAyAiGiLiqXR6K7ASmMUIOy59lKM3w7IcAJHYlr6sTh/BEB8TJ4sus4CXcl6vp+8v13ARwD2SFku6KJ23d0Q0QPJHA+yVzh8JZexv7LPS6fz5w8Flkp5Jm6k6mwhGTDkkzQGOJvlPdsQel7xywAg8LpIqJT0NbALujYghPyZOFl0Ktd2NhHHFJ0bEMcBpwKWSTupj3ZFaRug99uFapuuBA4GjgAbg6+n8EVEOSZOAnwGfjYimvlYtMG/YlKdAOUbkcYmI9og4CphNUks4rI/Vy1IWJ4su64F9c17PBjZkFEvJImJD+rwJuI2kWemVtMpJ+rwpXX0klLG/sa9Pp/PnZyoiXkn/wDuA79LV3DfsyyGpmuQH9kcR8fN09og7LoXKMZKPC0BEvAH8FljAEB8TJ4sui4CDJe0vqQY4D7gj45j6JGk3SZM7p4FTgGUkcV+YrnYhcHs6fQdwnqRxkvYHDibp8BpO+hV7Wv3eKun4dGTHx3Pek5nOP+LUWSTHBYZ5OdJ93wisjIhv5CwaUcelt3KMxOMiqVbStHR6AnAy8CxDfUyGsld/uD+A00lGTfwBuCLreEqI9wCSUQ9LgOWdMQN7AvcDz6fPe+S854q0fKvIYLRNXvw/IWkKaCX5r+eTuxI7UEfyR/8H4N9Ir0yQcTl+ACwFnkn/eGcM93KkMfwxSdPEM8DT6eP0kXZc+ijHiDsuwBHA79OYlwFXpvOH9Jj4ch9mZlaUm6HMzKwoJwszMyvKycLMzIpysjAzs6KcLMzMrCgnC7NBJumzkiZmHYfZYPLQWbNBJmktUBcRm7OOxWywuGZhNgDpWfR3pvcaWCbpi8BM4AFJD6TrnCLpMUlPSfrv9HpFnfciuTq9V8GTkg5K55+TbmuJpIeyK51ZFycLs4FZAGyIiCMj4jDgWpLr7bw3It4raTrwD8DJkVzwsR7465z3N0XEcSRn016bzrsSODUijgQ+NDTFMOubk4XZwCwFTk5rCO+KiC15y48nuRnN79JLTF8I7Jez/Cc5zyek078D/kvS/yG5KZdZ5qqyDsBsJIuI5yQdS3LdoS9LuidvFZHcf+D83jaRPx0RF0t6J/BB4GlJR0XEq4Mdu1l/uGZhNgCSZgLbI+KHwNdIbq+6leRWngCPAyfm9EdMlHRIzibOzXl+LF3nwIh4IiKuBDbT/XLTZplwzcJsYA4HrpHUQXLV2UtImpN+Jakh7bf4BPATSePS9/wDydWNAcZJeoLkH7fO2sc1kg4mqZXcT3JVYbNMeeisWUY8xNZGEjdDmZlZUa5ZmJlZUa5ZmJlZUU4WZmZWlJOFmZkV5WRhZmZFOVmYmVlR/x8XxShcebk1ZQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_scatter(list(range(3001)), ys, 'steps', 'y_inits')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21a90acf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "28e1626e",
   "metadata": {},
   "source": [
    "equ4.4Pricing of European.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "edc3e2db",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from scipy.stats import multivariate_normal as normal\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import *\n",
    "import time\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "tf.keras.backend.set_floatx('float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6dc9cdf6",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Y_compute(Layer):\n",
    "    def __init__(self,dimension):\n",
    "        self.dimension=dimension\n",
    "        self.y_init = tf.Variable(np.random.uniform(size=[1]))\n",
    "        self.grad_init = tf.Variable(\n",
    "            np.random.uniform(size=[1, self.dimension]))\n",
    "        super(Y_compute, self).__init__()\n",
    "\n",
    "    def call(self, inputs, **kwargs):\n",
    "        all_one_vec = tf.ones_like(tf.reduce_sum(tf.reduce_sum(inputs, axis=1), axis=1, keepdims=True))\n",
    "        y = all_one_vec * self.y_init\n",
    "        grad = tf.matmul(all_one_vec, self.grad_init)\n",
    "        return y,grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b73975c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BSDE():\n",
    "    def __init__(self):\n",
    "        self.dimension =100\n",
    "        self.T = 1\n",
    "        self.partion = 20\n",
    "        self.interval = self.T / self.partion\n",
    "        self.sqrt_interval = np.sqrt(self.interval)\n",
    "        #Specifically\n",
    "        #self.lambd = 1.0\n",
    "        self.time_stamp = np.arange(0, self.partion) * self.interval\n",
    "        #self.x_init = np.ones(self.dimension) * 100\n",
    "        self.x_init = np.zeros(self.dimension)\n",
    "        self.sigma = 0.2\n",
    "        self.mu_bar = 0.06\n",
    "        self.rl = 0.04\n",
    "        self.rb = 0.06\n",
    "        self.alpha = 1.0 / self.dimension\n",
    "\n",
    "    # f in the equation\n",
    "    def f_equ(self, t, x, y, grad):\n",
    "        temp = tf.reduce_sum(grad, 1, keepdims=True) / self.sigma\n",
    "        return -self.rl * y - (self.mu_bar - self.rl) * temp + (\n",
    "                (self.rb - self.rl) * tf.maximum(temp - y, 0))\n",
    "\n",
    "    # Final\n",
    "    def g_final(self, t, x):\n",
    "        temp = tf.reduce_max(x, 1, keepdims=True)\n",
    "        return tf.maximum(temp - 120, 0) - 2 * tf.maximum(temp - 150, 0)\n",
    "\n",
    "    # Data generator with motion of X\n",
    "    def sample(self, num_sample):\n",
    "        dw_sample = normal.rvs(size=[num_sample, self.dimension,\n",
    "                                     self.partion]) * self.sqrt_interval\n",
    "        x_sample = np.zeros([num_sample, self.dimension, self.partion + 1])\n",
    "        x_sample[:, :, 0] = np.ones([num_sample, self.dimension]) * self.x_init\n",
    "        factor = np.exp((self.mu_bar-(self.sigma**2)/2)*self.interval)\n",
    "        for i in range(self.partion):\n",
    "            x_sample[:, :, i + 1] = (factor * np.exp(self.sigma * dw_sample[:,:,i])) * x_sample[:,:,i]\n",
    "\n",
    "        y_true=self.g_final(self.T, x_sample[:, :, -1])\n",
    "        return x_sample,dw_sample,y_true\n",
    "\n",
    "\n",
    "    def part(self,x,num_hid=110,dup=2):\n",
    "        for i in range(dup):\n",
    "            x=Dense(num_hid,activation='relu')(x)\n",
    "            x=BatchNormalization(\n",
    "                momentum=0.99,\n",
    "                epsilon=1e-6,\n",
    "                beta_initializer=tf.random_normal_initializer(0.0, stddev=0.1),\n",
    "                gamma_initializer=tf.random_uniform_initializer(0.1, 0.5)\n",
    "            )(x)\n",
    "        x = Dense(self.dimension, activation='relu')(x)\n",
    "        return x\n",
    "\n",
    "    def init_model(self,):\n",
    "        x1=Input(shape=(self.dimension,self.partion+1))\n",
    "        dw=Input(shape=(self.dimension,self.partion))\n",
    "\n",
    "\n",
    "        y,grad=Y_compute(self.dimension)(x1)\n",
    "\n",
    "        for t in range(0, self.partion-1):\n",
    "            y = y - self.interval * (\n",
    "                self.f_equ(self.time_stamp[t], x1[:, :, t], y, grad))\n",
    "            grad = self.part(x1[:, :, t+1])\n",
    "\n",
    "        # terminal time\n",
    "        out = y - self.interval * self.f_equ(self.time_stamp[-1], x1[:, :, -2], y, grad)\n",
    "\n",
    "        model=Model(inputs=[x1,dw],outputs=out)\n",
    "        model.compile(loss='mse',optimizer='adam')\n",
    "        return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d6abadd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_scatter(x,y,x_name,y_name):\n",
    "    # 创建画图窗口\n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.set_title('{} with {}'.format(y_name,x_name))\n",
    "    ax.set_xlabel(x_name)\n",
    "    ax.set_ylabel(y_name)\n",
    "    ax.scatter(x, y,marker='.',s=1)\n",
    "    plt.show()\n",
    "    # plt.savefig(r'{} with {}.jpg'.format(y_name,x_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ac6bc857",
   "metadata": {},
   "outputs": [],
   "source": [
    "bsde=BSDE()\n",
    "model=bsde.init_model()\n",
    "x1_valid,x2_valid,y_valid = bsde.sample(256)\n",
    "# print(x1_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "35700919",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 100, 21)]    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "y_compute (Y_compute)           ((None, 1), (None, 1 101         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum (TFOpLambda) (None, 1)            0           y_compute[0][1]                  \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_1 (Sli (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv (TFOpLambda)    (None, 1)            0           tf.math.reduce_sum[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 110)          11110       tf.__operators__.getitem_1[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_1 (TFOpLambda) (None, 1)            0           tf.math.truediv[0][0]            \n",
      "                                                                 y_compute[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 110)          440         dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply (TFOpLambda)   (None, 1)            0           y_compute[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_1 (TFOpLambda) (None, 1)            0           tf.math.truediv[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum (TFOpLambda)    (None, 1)            0           tf.math.subtract_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 110)          12210       batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract (TFOpLambda)   (None, 1)            0           tf.math.multiply[0][0]           \n",
      "                                                                 tf.math.multiply_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_2 (TFOpLambda) (None, 1)            0           tf.math.maximum[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 110)          440         dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add (TFOpLambd (None, 1)            0           tf.math.subtract[0][0]           \n",
      "                                                                 tf.math.multiply_2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 100)          11100       batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_3 (TFOpLambda) (None, 1)            0           tf.__operators__.add[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_1 (TFOpLambd (None, 1)            0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_3 (Sli (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_2 (TFOpLambda) (None, 1)            0           y_compute[0][0]                  \n",
      "                                                                 tf.math.multiply_3[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_1 (TFOpLambda)  (None, 1)            0           tf.math.reduce_sum_1[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 110)          11110       tf.__operators__.getitem_3[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_4 (TFOpLambda) (None, 1)            0           tf.math.truediv_1[0][0]          \n",
      "                                                                 tf.math.subtract_2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 110)          440         dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_4 (TFOpLambda) (None, 1)            0           tf.math.subtract_2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_5 (TFOpLambda) (None, 1)            0           tf.math.truediv_1[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_1 (TFOpLambda)  (None, 1)            0           tf.math.subtract_4[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 110)          12210       batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_3 (TFOpLambda) (None, 1)            0           tf.math.multiply_4[0][0]         \n",
      "                                                                 tf.math.multiply_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_6 (TFOpLambda) (None, 1)            0           tf.math.maximum_1[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 110)          440         dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_1 (TFOpLam (None, 1)            0           tf.math.subtract_3[0][0]         \n",
      "                                                                 tf.math.multiply_6[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 100)          11100       batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_7 (TFOpLambda) (None, 1)            0           tf.__operators__.add_1[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_2 (TFOpLambd (None, 1)            0           dense_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_5 (Sli (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_5 (TFOpLambda) (None, 1)            0           tf.math.subtract_2[0][0]         \n",
      "                                                                 tf.math.multiply_7[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_2 (TFOpLambda)  (None, 1)            0           tf.math.reduce_sum_2[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 110)          11110       tf.__operators__.getitem_5[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_7 (TFOpLambda) (None, 1)            0           tf.math.truediv_2[0][0]          \n",
      "                                                                 tf.math.subtract_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 110)          440         dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_8 (TFOpLambda) (None, 1)            0           tf.math.subtract_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_9 (TFOpLambda) (None, 1)            0           tf.math.truediv_2[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_2 (TFOpLambda)  (None, 1)            0           tf.math.subtract_7[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 110)          12210       batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_6 (TFOpLambda) (None, 1)            0           tf.math.multiply_8[0][0]         \n",
      "                                                                 tf.math.multiply_9[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_10 (TFOpLambda (None, 1)            0           tf.math.maximum_2[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 110)          440         dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_2 (TFOpLam (None, 1)            0           tf.math.subtract_6[0][0]         \n",
      "                                                                 tf.math.multiply_10[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 100)          11100       batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_11 (TFOpLambda (None, 1)            0           tf.__operators__.add_2[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_3 (TFOpLambd (None, 1)            0           dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_7 (Sli (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_8 (TFOpLambda) (None, 1)            0           tf.math.subtract_5[0][0]         \n",
      "                                                                 tf.math.multiply_11[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_3 (TFOpLambda)  (None, 1)            0           tf.math.reduce_sum_3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 110)          11110       tf.__operators__.getitem_7[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_10 (TFOpLambda (None, 1)            0           tf.math.truediv_3[0][0]          \n",
      "                                                                 tf.math.subtract_8[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 110)          440         dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_12 (TFOpLambda (None, 1)            0           tf.math.subtract_8[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_13 (TFOpLambda (None, 1)            0           tf.math.truediv_3[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_3 (TFOpLambda)  (None, 1)            0           tf.math.subtract_10[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 110)          12210       batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_9 (TFOpLambda) (None, 1)            0           tf.math.multiply_12[0][0]        \n",
      "                                                                 tf.math.multiply_13[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_14 (TFOpLambda (None, 1)            0           tf.math.maximum_3[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 110)          440         dense_10[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_3 (TFOpLam (None, 1)            0           tf.math.subtract_9[0][0]         \n",
      "                                                                 tf.math.multiply_14[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 100)          11100       batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_15 (TFOpLambda (None, 1)            0           tf.__operators__.add_3[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_4 (TFOpLambd (None, 1)            0           dense_11[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_9 (Sli (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_11 (TFOpLambda (None, 1)            0           tf.math.subtract_8[0][0]         \n",
      "                                                                 tf.math.multiply_15[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_4 (TFOpLambda)  (None, 1)            0           tf.math.reduce_sum_4[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 110)          11110       tf.__operators__.getitem_9[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_13 (TFOpLambda (None, 1)            0           tf.math.truediv_4[0][0]          \n",
      "                                                                 tf.math.subtract_11[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 110)          440         dense_12[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_16 (TFOpLambda (None, 1)            0           tf.math.subtract_11[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_17 (TFOpLambda (None, 1)            0           tf.math.truediv_4[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_4 (TFOpLambda)  (None, 1)            0           tf.math.subtract_13[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 110)          12210       batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_12 (TFOpLambda (None, 1)            0           tf.math.multiply_16[0][0]        \n",
      "                                                                 tf.math.multiply_17[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_18 (TFOpLambda (None, 1)            0           tf.math.maximum_4[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 110)          440         dense_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_4 (TFOpLam (None, 1)            0           tf.math.subtract_12[0][0]        \n",
      "                                                                 tf.math.multiply_18[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (None, 100)          11100       batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_19 (TFOpLambda (None, 1)            0           tf.__operators__.add_4[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_5 (TFOpLambd (None, 1)            0           dense_14[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_11 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_14 (TFOpLambda (None, 1)            0           tf.math.subtract_11[0][0]        \n",
      "                                                                 tf.math.multiply_19[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_5 (TFOpLambda)  (None, 1)            0           tf.math.reduce_sum_5[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 110)          11110       tf.__operators__.getitem_11[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_16 (TFOpLambda (None, 1)            0           tf.math.truediv_5[0][0]          \n",
      "                                                                 tf.math.subtract_14[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 110)          440         dense_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_20 (TFOpLambda (None, 1)            0           tf.math.subtract_14[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_21 (TFOpLambda (None, 1)            0           tf.math.truediv_5[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_5 (TFOpLambda)  (None, 1)            0           tf.math.subtract_16[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 110)          12210       batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_15 (TFOpLambda (None, 1)            0           tf.math.multiply_20[0][0]        \n",
      "                                                                 tf.math.multiply_21[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_22 (TFOpLambda (None, 1)            0           tf.math.maximum_5[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 110)          440         dense_16[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_5 (TFOpLam (None, 1)            0           tf.math.subtract_15[0][0]        \n",
      "                                                                 tf.math.multiply_22[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 100)          11100       batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_23 (TFOpLambda (None, 1)            0           tf.__operators__.add_5[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_6 (TFOpLambd (None, 1)            0           dense_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_13 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_17 (TFOpLambda (None, 1)            0           tf.math.subtract_14[0][0]        \n",
      "                                                                 tf.math.multiply_23[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_6 (TFOpLambda)  (None, 1)            0           tf.math.reduce_sum_6[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_18 (Dense)                (None, 110)          11110       tf.__operators__.getitem_13[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_19 (TFOpLambda (None, 1)            0           tf.math.truediv_6[0][0]          \n",
      "                                                                 tf.math.subtract_17[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 110)          440         dense_18[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_24 (TFOpLambda (None, 1)            0           tf.math.subtract_17[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_25 (TFOpLambda (None, 1)            0           tf.math.truediv_6[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_6 (TFOpLambda)  (None, 1)            0           tf.math.subtract_19[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_19 (Dense)                (None, 110)          12210       batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_18 (TFOpLambda (None, 1)            0           tf.math.multiply_24[0][0]        \n",
      "                                                                 tf.math.multiply_25[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_26 (TFOpLambda (None, 1)            0           tf.math.maximum_6[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 110)          440         dense_19[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_6 (TFOpLam (None, 1)            0           tf.math.subtract_18[0][0]        \n",
      "                                                                 tf.math.multiply_26[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_20 (Dense)                (None, 100)          11100       batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_27 (TFOpLambda (None, 1)            0           tf.__operators__.add_6[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_7 (TFOpLambd (None, 1)            0           dense_20[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_15 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_20 (TFOpLambda (None, 1)            0           tf.math.subtract_17[0][0]        \n",
      "                                                                 tf.math.multiply_27[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_7 (TFOpLambda)  (None, 1)            0           tf.math.reduce_sum_7[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_21 (Dense)                (None, 110)          11110       tf.__operators__.getitem_15[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_22 (TFOpLambda (None, 1)            0           tf.math.truediv_7[0][0]          \n",
      "                                                                 tf.math.subtract_20[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 110)          440         dense_21[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_28 (TFOpLambda (None, 1)            0           tf.math.subtract_20[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_29 (TFOpLambda (None, 1)            0           tf.math.truediv_7[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_7 (TFOpLambda)  (None, 1)            0           tf.math.subtract_22[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_22 (Dense)                (None, 110)          12210       batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_21 (TFOpLambda (None, 1)            0           tf.math.multiply_28[0][0]        \n",
      "                                                                 tf.math.multiply_29[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_30 (TFOpLambda (None, 1)            0           tf.math.maximum_7[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 110)          440         dense_22[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_7 (TFOpLam (None, 1)            0           tf.math.subtract_21[0][0]        \n",
      "                                                                 tf.math.multiply_30[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_23 (Dense)                (None, 100)          11100       batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_31 (TFOpLambda (None, 1)            0           tf.__operators__.add_7[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_8 (TFOpLambd (None, 1)            0           dense_23[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_17 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_23 (TFOpLambda (None, 1)            0           tf.math.subtract_20[0][0]        \n",
      "                                                                 tf.math.multiply_31[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_8 (TFOpLambda)  (None, 1)            0           tf.math.reduce_sum_8[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_24 (Dense)                (None, 110)          11110       tf.__operators__.getitem_17[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_25 (TFOpLambda (None, 1)            0           tf.math.truediv_8[0][0]          \n",
      "                                                                 tf.math.subtract_23[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 110)          440         dense_24[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_32 (TFOpLambda (None, 1)            0           tf.math.subtract_23[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_33 (TFOpLambda (None, 1)            0           tf.math.truediv_8[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_8 (TFOpLambda)  (None, 1)            0           tf.math.subtract_25[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_25 (Dense)                (None, 110)          12210       batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_24 (TFOpLambda (None, 1)            0           tf.math.multiply_32[0][0]        \n",
      "                                                                 tf.math.multiply_33[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_34 (TFOpLambda (None, 1)            0           tf.math.maximum_8[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 110)          440         dense_25[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_8 (TFOpLam (None, 1)            0           tf.math.subtract_24[0][0]        \n",
      "                                                                 tf.math.multiply_34[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_26 (Dense)                (None, 100)          11100       batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_35 (TFOpLambda (None, 1)            0           tf.__operators__.add_8[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_9 (TFOpLambd (None, 1)            0           dense_26[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_19 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_26 (TFOpLambda (None, 1)            0           tf.math.subtract_23[0][0]        \n",
      "                                                                 tf.math.multiply_35[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_9 (TFOpLambda)  (None, 1)            0           tf.math.reduce_sum_9[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_27 (Dense)                (None, 110)          11110       tf.__operators__.getitem_19[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_28 (TFOpLambda (None, 1)            0           tf.math.truediv_9[0][0]          \n",
      "                                                                 tf.math.subtract_26[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 110)          440         dense_27[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_36 (TFOpLambda (None, 1)            0           tf.math.subtract_26[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_37 (TFOpLambda (None, 1)            0           tf.math.truediv_9[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_9 (TFOpLambda)  (None, 1)            0           tf.math.subtract_28[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_28 (Dense)                (None, 110)          12210       batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_27 (TFOpLambda (None, 1)            0           tf.math.multiply_36[0][0]        \n",
      "                                                                 tf.math.multiply_37[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_38 (TFOpLambda (None, 1)            0           tf.math.maximum_9[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, 110)          440         dense_28[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_9 (TFOpLam (None, 1)            0           tf.math.subtract_27[0][0]        \n",
      "                                                                 tf.math.multiply_38[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_29 (Dense)                (None, 100)          11100       batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_39 (TFOpLambda (None, 1)            0           tf.__operators__.add_9[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_10 (TFOpLamb (None, 1)            0           dense_29[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_21 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_29 (TFOpLambda (None, 1)            0           tf.math.subtract_26[0][0]        \n",
      "                                                                 tf.math.multiply_39[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_10 (TFOpLambda) (None, 1)            0           tf.math.reduce_sum_10[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_30 (Dense)                (None, 110)          11110       tf.__operators__.getitem_21[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_31 (TFOpLambda (None, 1)            0           tf.math.truediv_10[0][0]         \n",
      "                                                                 tf.math.subtract_29[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNo (None, 110)          440         dense_30[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_40 (TFOpLambda (None, 1)            0           tf.math.subtract_29[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_41 (TFOpLambda (None, 1)            0           tf.math.truediv_10[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_10 (TFOpLambda) (None, 1)            0           tf.math.subtract_31[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_31 (Dense)                (None, 110)          12210       batch_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_30 (TFOpLambda (None, 1)            0           tf.math.multiply_40[0][0]        \n",
      "                                                                 tf.math.multiply_41[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_42 (TFOpLambda (None, 1)            0           tf.math.maximum_10[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNo (None, 110)          440         dense_31[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_10 (TFOpLa (None, 1)            0           tf.math.subtract_30[0][0]        \n",
      "                                                                 tf.math.multiply_42[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_32 (Dense)                (None, 100)          11100       batch_normalization_21[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_43 (TFOpLambda (None, 1)            0           tf.__operators__.add_10[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_11 (TFOpLamb (None, 1)            0           dense_32[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_23 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_32 (TFOpLambda (None, 1)            0           tf.math.subtract_29[0][0]        \n",
      "                                                                 tf.math.multiply_43[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_11 (TFOpLambda) (None, 1)            0           tf.math.reduce_sum_11[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_33 (Dense)                (None, 110)          11110       tf.__operators__.getitem_23[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_34 (TFOpLambda (None, 1)            0           tf.math.truediv_11[0][0]         \n",
      "                                                                 tf.math.subtract_32[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNo (None, 110)          440         dense_33[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_44 (TFOpLambda (None, 1)            0           tf.math.subtract_32[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_45 (TFOpLambda (None, 1)            0           tf.math.truediv_11[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_11 (TFOpLambda) (None, 1)            0           tf.math.subtract_34[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_34 (Dense)                (None, 110)          12210       batch_normalization_22[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_33 (TFOpLambda (None, 1)            0           tf.math.multiply_44[0][0]        \n",
      "                                                                 tf.math.multiply_45[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_46 (TFOpLambda (None, 1)            0           tf.math.maximum_11[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNo (None, 110)          440         dense_34[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_11 (TFOpLa (None, 1)            0           tf.math.subtract_33[0][0]        \n",
      "                                                                 tf.math.multiply_46[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_35 (Dense)                (None, 100)          11100       batch_normalization_23[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_47 (TFOpLambda (None, 1)            0           tf.__operators__.add_11[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_12 (TFOpLamb (None, 1)            0           dense_35[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_25 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_35 (TFOpLambda (None, 1)            0           tf.math.subtract_32[0][0]        \n",
      "                                                                 tf.math.multiply_47[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_12 (TFOpLambda) (None, 1)            0           tf.math.reduce_sum_12[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_36 (Dense)                (None, 110)          11110       tf.__operators__.getitem_25[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_37 (TFOpLambda (None, 1)            0           tf.math.truediv_12[0][0]         \n",
      "                                                                 tf.math.subtract_35[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNo (None, 110)          440         dense_36[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_48 (TFOpLambda (None, 1)            0           tf.math.subtract_35[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_49 (TFOpLambda (None, 1)            0           tf.math.truediv_12[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_12 (TFOpLambda) (None, 1)            0           tf.math.subtract_37[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_37 (Dense)                (None, 110)          12210       batch_normalization_24[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_36 (TFOpLambda (None, 1)            0           tf.math.multiply_48[0][0]        \n",
      "                                                                 tf.math.multiply_49[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_50 (TFOpLambda (None, 1)            0           tf.math.maximum_12[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_25 (BatchNo (None, 110)          440         dense_37[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_12 (TFOpLa (None, 1)            0           tf.math.subtract_36[0][0]        \n",
      "                                                                 tf.math.multiply_50[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_38 (Dense)                (None, 100)          11100       batch_normalization_25[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_51 (TFOpLambda (None, 1)            0           tf.__operators__.add_12[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_13 (TFOpLamb (None, 1)            0           dense_38[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_27 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_38 (TFOpLambda (None, 1)            0           tf.math.subtract_35[0][0]        \n",
      "                                                                 tf.math.multiply_51[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_13 (TFOpLambda) (None, 1)            0           tf.math.reduce_sum_13[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_39 (Dense)                (None, 110)          11110       tf.__operators__.getitem_27[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_40 (TFOpLambda (None, 1)            0           tf.math.truediv_13[0][0]         \n",
      "                                                                 tf.math.subtract_38[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_26 (BatchNo (None, 110)          440         dense_39[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_52 (TFOpLambda (None, 1)            0           tf.math.subtract_38[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_53 (TFOpLambda (None, 1)            0           tf.math.truediv_13[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_13 (TFOpLambda) (None, 1)            0           tf.math.subtract_40[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_40 (Dense)                (None, 110)          12210       batch_normalization_26[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_39 (TFOpLambda (None, 1)            0           tf.math.multiply_52[0][0]        \n",
      "                                                                 tf.math.multiply_53[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_54 (TFOpLambda (None, 1)            0           tf.math.maximum_13[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_27 (BatchNo (None, 110)          440         dense_40[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_13 (TFOpLa (None, 1)            0           tf.math.subtract_39[0][0]        \n",
      "                                                                 tf.math.multiply_54[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_41 (Dense)                (None, 100)          11100       batch_normalization_27[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_55 (TFOpLambda (None, 1)            0           tf.__operators__.add_13[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_14 (TFOpLamb (None, 1)            0           dense_41[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_29 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_41 (TFOpLambda (None, 1)            0           tf.math.subtract_38[0][0]        \n",
      "                                                                 tf.math.multiply_55[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_14 (TFOpLambda) (None, 1)            0           tf.math.reduce_sum_14[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_42 (Dense)                (None, 110)          11110       tf.__operators__.getitem_29[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_43 (TFOpLambda (None, 1)            0           tf.math.truediv_14[0][0]         \n",
      "                                                                 tf.math.subtract_41[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_28 (BatchNo (None, 110)          440         dense_42[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_56 (TFOpLambda (None, 1)            0           tf.math.subtract_41[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_57 (TFOpLambda (None, 1)            0           tf.math.truediv_14[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_14 (TFOpLambda) (None, 1)            0           tf.math.subtract_43[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_43 (Dense)                (None, 110)          12210       batch_normalization_28[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_42 (TFOpLambda (None, 1)            0           tf.math.multiply_56[0][0]        \n",
      "                                                                 tf.math.multiply_57[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_58 (TFOpLambda (None, 1)            0           tf.math.maximum_14[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_29 (BatchNo (None, 110)          440         dense_43[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_14 (TFOpLa (None, 1)            0           tf.math.subtract_42[0][0]        \n",
      "                                                                 tf.math.multiply_58[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_44 (Dense)                (None, 100)          11100       batch_normalization_29[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_59 (TFOpLambda (None, 1)            0           tf.__operators__.add_14[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_15 (TFOpLamb (None, 1)            0           dense_44[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_31 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_44 (TFOpLambda (None, 1)            0           tf.math.subtract_41[0][0]        \n",
      "                                                                 tf.math.multiply_59[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_15 (TFOpLambda) (None, 1)            0           tf.math.reduce_sum_15[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_45 (Dense)                (None, 110)          11110       tf.__operators__.getitem_31[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_46 (TFOpLambda (None, 1)            0           tf.math.truediv_15[0][0]         \n",
      "                                                                 tf.math.subtract_44[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_30 (BatchNo (None, 110)          440         dense_45[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_60 (TFOpLambda (None, 1)            0           tf.math.subtract_44[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_61 (TFOpLambda (None, 1)            0           tf.math.truediv_15[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_15 (TFOpLambda) (None, 1)            0           tf.math.subtract_46[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_46 (Dense)                (None, 110)          12210       batch_normalization_30[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_45 (TFOpLambda (None, 1)            0           tf.math.multiply_60[0][0]        \n",
      "                                                                 tf.math.multiply_61[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_62 (TFOpLambda (None, 1)            0           tf.math.maximum_15[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_31 (BatchNo (None, 110)          440         dense_46[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_15 (TFOpLa (None, 1)            0           tf.math.subtract_45[0][0]        \n",
      "                                                                 tf.math.multiply_62[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_47 (Dense)                (None, 100)          11100       batch_normalization_31[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_63 (TFOpLambda (None, 1)            0           tf.__operators__.add_15[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_16 (TFOpLamb (None, 1)            0           dense_47[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_33 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_47 (TFOpLambda (None, 1)            0           tf.math.subtract_44[0][0]        \n",
      "                                                                 tf.math.multiply_63[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_16 (TFOpLambda) (None, 1)            0           tf.math.reduce_sum_16[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_48 (Dense)                (None, 110)          11110       tf.__operators__.getitem_33[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_49 (TFOpLambda (None, 1)            0           tf.math.truediv_16[0][0]         \n",
      "                                                                 tf.math.subtract_47[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_32 (BatchNo (None, 110)          440         dense_48[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_64 (TFOpLambda (None, 1)            0           tf.math.subtract_47[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_65 (TFOpLambda (None, 1)            0           tf.math.truediv_16[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_16 (TFOpLambda) (None, 1)            0           tf.math.subtract_49[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_49 (Dense)                (None, 110)          12210       batch_normalization_32[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_48 (TFOpLambda (None, 1)            0           tf.math.multiply_64[0][0]        \n",
      "                                                                 tf.math.multiply_65[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_66 (TFOpLambda (None, 1)            0           tf.math.maximum_16[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_33 (BatchNo (None, 110)          440         dense_49[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_16 (TFOpLa (None, 1)            0           tf.math.subtract_48[0][0]        \n",
      "                                                                 tf.math.multiply_66[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_50 (Dense)                (None, 100)          11100       batch_normalization_33[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_67 (TFOpLambda (None, 1)            0           tf.__operators__.add_16[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_17 (TFOpLamb (None, 1)            0           dense_50[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_35 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_50 (TFOpLambda (None, 1)            0           tf.math.subtract_47[0][0]        \n",
      "                                                                 tf.math.multiply_67[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_17 (TFOpLambda) (None, 1)            0           tf.math.reduce_sum_17[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_51 (Dense)                (None, 110)          11110       tf.__operators__.getitem_35[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_52 (TFOpLambda (None, 1)            0           tf.math.truediv_17[0][0]         \n",
      "                                                                 tf.math.subtract_50[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_34 (BatchNo (None, 110)          440         dense_51[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_68 (TFOpLambda (None, 1)            0           tf.math.subtract_50[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_69 (TFOpLambda (None, 1)            0           tf.math.truediv_17[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_17 (TFOpLambda) (None, 1)            0           tf.math.subtract_52[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_52 (Dense)                (None, 110)          12210       batch_normalization_34[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_51 (TFOpLambda (None, 1)            0           tf.math.multiply_68[0][0]        \n",
      "                                                                 tf.math.multiply_69[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_70 (TFOpLambda (None, 1)            0           tf.math.maximum_17[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_35 (BatchNo (None, 110)          440         dense_52[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_17 (TFOpLa (None, 1)            0           tf.math.subtract_51[0][0]        \n",
      "                                                                 tf.math.multiply_70[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_53 (Dense)                (None, 100)          11100       batch_normalization_35[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_71 (TFOpLambda (None, 1)            0           tf.__operators__.add_17[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_18 (TFOpLamb (None, 1)            0           dense_53[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.getitem_37 (Sl (None, 100)          0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_53 (TFOpLambda (None, 1)            0           tf.math.subtract_50[0][0]        \n",
      "                                                                 tf.math.multiply_71[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_18 (TFOpLambda) (None, 1)            0           tf.math.reduce_sum_18[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_54 (Dense)                (None, 110)          11110       tf.__operators__.getitem_37[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_55 (TFOpLambda (None, 1)            0           tf.math.truediv_18[0][0]         \n",
      "                                                                 tf.math.subtract_53[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_36 (BatchNo (None, 110)          440         dense_54[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_72 (TFOpLambda (None, 1)            0           tf.math.subtract_53[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_73 (TFOpLambda (None, 1)            0           tf.math.truediv_18[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_18 (TFOpLambda) (None, 1)            0           tf.math.subtract_55[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_55 (Dense)                (None, 110)          12210       batch_normalization_36[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_54 (TFOpLambda (None, 1)            0           tf.math.multiply_72[0][0]        \n",
      "                                                                 tf.math.multiply_73[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_74 (TFOpLambda (None, 1)            0           tf.math.maximum_18[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_37 (BatchNo (None, 110)          440         dense_55[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_18 (TFOpLa (None, 1)            0           tf.math.subtract_54[0][0]        \n",
      "                                                                 tf.math.multiply_74[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_56 (Dense)                (None, 100)          11100       batch_normalization_37[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_75 (TFOpLambda (None, 1)            0           tf.__operators__.add_18[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.reduce_sum_19 (TFOpLamb (None, 1)            0           dense_56[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_56 (TFOpLambda (None, 1)            0           tf.math.subtract_53[0][0]        \n",
      "                                                                 tf.math.multiply_75[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.truediv_19 (TFOpLambda) (None, 1)            0           tf.math.reduce_sum_19[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_58 (TFOpLambda (None, 1)            0           tf.math.truediv_19[0][0]         \n",
      "                                                                 tf.math.subtract_56[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_76 (TFOpLambda (None, 1)            0           tf.math.subtract_56[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_77 (TFOpLambda (None, 1)            0           tf.math.truediv_19[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.maximum_19 (TFOpLambda) (None, 1)            0           tf.math.subtract_58[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_57 (TFOpLambda (None, 1)            0           tf.math.multiply_76[0][0]        \n",
      "                                                                 tf.math.multiply_77[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_78 (TFOpLambda (None, 1)            0           tf.math.maximum_19[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.__operators__.add_19 (TFOpLa (None, 1)            0           tf.math.subtract_57[0][0]        \n",
      "                                                                 tf.math.multiply_78[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.multiply_79 (TFOpLambda (None, 1)            0           tf.__operators__.add_19[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            [(None, 100, 20)]    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf.math.subtract_59 (TFOpLambda (None, 1)            0           tf.math.subtract_56[0][0]        \n",
      "                                                                 tf.math.multiply_79[0][0]        \n",
      "==================================================================================================\n",
      "Total params: 670,801\n",
      "Trainable params: 662,441\n",
      "Non-trainable params: 8,360\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f0927a14",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# tf.keras.utils.plot_model(model, 'model.png', show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "be156123",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step:     0,    loss: 6.6036e-03, Y_init: 7.6537e-02,   elapsed time:  11\n",
      "step:   100,    loss: 6.9330e-05, Y_init: 7.8423e-03,   elapsed time:  19\n",
      "step:   200,    loss: 6.5531e-11, Y_init: 7.6244e-06,   elapsed time:  27\n",
      "step:   300,    loss: 3.6176e-14, Y_init: 1.7914e-07,   elapsed time:  35\n",
      "step:   400,    loss: 1.3086e-18, Y_init: -1.0774e-09,   elapsed time:  43\n",
      "step:   500,    loss: 7.0929e-23, Y_init: -7.9322e-12,   elapsed time:  52\n",
      "step:   600,    loss: 1.8934e-27, Y_init: -4.0965e-14,   elapsed time:  60\n",
      "step:   700,    loss: 2.8507e-32, Y_init: -1.6884e-16,   elapsed time:  69\n",
      "step:   800,    loss: 7.3954e-41, Y_init: 8.5997e-21,   elapsed time:  77\n",
      "step:   900,    loss: 3.6505e-41, Y_init: 6.0419e-21,   elapsed time:  85\n",
      "step:  1000,    loss: 7.6472e-49, Y_init: -8.7448e-25,   elapsed time:  94\n",
      "step:  1100,    loss: 1.5670e-50, Y_init: -1.2518e-25,   elapsed time: 102\n",
      "step:  1200,    loss: 6.9901e-55, Y_init: 8.3607e-28,   elapsed time: 111\n",
      "step:  1300,    loss: 1.8817e-59, Y_init: -4.3378e-30,   elapsed time: 119\n",
      "step:  1400,    loss: 2.9138e-64, Y_init: 1.7070e-32,   elapsed time: 128\n",
      "step:  1500,    loss: 9.0029e-71, Y_init: 9.4884e-36,   elapsed time: 136\n",
      "step:  1600,    loss: 4.0698e-73, Y_init: -6.3795e-37,   elapsed time: 145\n",
      "step:  1700,    loss: 4.0500e-80, Y_init: 2.0125e-40,   elapsed time: 153\n",
      "step:  1800,    loss: 2.4053e-82, Y_init: 1.5509e-41,   elapsed time: 162\n",
      "step:  1900,    loss: 8.4918e-87, Y_init: 9.2151e-44,   elapsed time: 171\n",
      "step:  2000,    loss: 2.1964e-91, Y_init: 4.6865e-46,   elapsed time: 180\n",
      "step:  2100,    loss: 6.3192e-96, Y_init: 2.5138e-48,   elapsed time: 189\n",
      "step:  2200,    loss: 9.4798e-101, Y_init: 9.7364e-51,   elapsed time: 197\n",
      "step:  2300,    loss: 4.1699e-106, Y_init: -2.0420e-53,   elapsed time: 206\n",
      "step:  2400,    loss: 1.1523e-109, Y_init: -3.3946e-55,   elapsed time: 215\n",
      "step:  2500,    loss: 1.6929e-114, Y_init: 1.3011e-57,   elapsed time: 223\n",
      "step:  2600,    loss: 1.3045e-120, Y_init: -1.1421e-60,   elapsed time: 232\n",
      "step:  2700,    loss: 1.4674e-125, Y_init: -3.8307e-63,   elapsed time: 241\n",
      "step:  2800,    loss: 2.7745e-129, Y_init: -5.2673e-65,   elapsed time: 249\n",
      "step:  2900,    loss: 1.2661e-132, Y_init: 1.1252e-66,   elapsed time: 258\n",
      "step:  3000,    loss: 3.3939e-137, Y_init: -5.8258e-69,   elapsed time: 267\n",
      "Done!!\n"
     ]
    }
   ],
   "source": [
    "losses,ys=[],[]\n",
    "start_time = time.time()\n",
    "\n",
    "# begin SGD iteration\n",
    "for step in range(3001):\n",
    "    x1_train,x2_train,y_train = bsde.sample(64)\n",
    "    model.train_on_batch([x1_train,x2_train], y_train)\n",
    "\n",
    "    y_pred = model.predict_on_batch([x1_valid, x2_valid])\n",
    "    loss = mean_squared_error(y_valid, y_pred)\n",
    "    y_init=model.get_layer('y_compute').get_weights()[0]\n",
    "    if step % 100 == 0:  # trace in terminal\n",
    "        elapsed_time = time.time() - start_time\n",
    "        print(\"step: %5u,    loss: %.4e, Y_init: %.4e,   elapsed time: %3u\" %\n",
    "              (step, loss, y_init, elapsed_time))\n",
    "    losses.append(round(loss,4))\n",
    "    ys.append(np.round(y_init, 4))\n",
    "print('Done!!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "74b46b56",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAdyElEQVR4nO3de5hcVZnv8e+PhkRRAiKN5gZJIF6iQk5sIR6FGRQ1yXDMcWZUODogXkJ8iI+O49F48DB4e0ZlxguKBNQoDErUmWEmYjSgo5CjBtJhIJBgsInBNGkhqCTh5EgSeM8fe/V0daW6au9O767q1O/zPPX0vqxVtVZX0m+td6+9ShGBmZlZXoc1uwFmZja2OHCYmVkhDhxmZlaIA4eZmRXiwGFmZoU4cJiZWSEOHDYmSdoq6exmt2MkSHpc0ow65w+ZvtqhwYHDrMki4pkRsQVA0jckfaKM15EUkk4u47mtvThwmJlZIQ4cNuZJGi/p85K2p8fnJY1P546TdJOkxyT9XtIaSYelcx+S9JCk3ZI2S3p1On6YpKWSHpD0O0nfkXRsOvc0Sden449JWifpOTXadKGk71Xs90j6TsX+Nkmz03ZIOlnSIuAtwAdT+up7FU85W9IGSTslfVvS04b4XZws6dZU7lFJ307Hb0tF7k7P/eZ0/BxJd6W+/FzSKRXPtVXShyVtkvQHSV/vf916v1drAxHhhx9j7gFsBc5O2x8D1gLHA53Az4GPp3N/BywDjkiPMwABzwe2AZNSuWnASWn7fen5pgDjgauBG9K5i4DvAUcCHcBLgQk12jcDeIzsw9lE4EHgoYpzfwAOS/sBnJy2vwF8okZf7wAmAccC9wGLh/i93ABckl73acArK8795+uk/TnAI8DpqS8XpNcaX/G69wJT0+v+rL9tQ/1em/3vwo/RefgTgh0K3gJ8LCIeiYgdwEeBv0rn9pH94T4xIvZFxJqICOBJsqAwS9IREbE1Ih5IdS4CLomI3oh4ArgM+EtJh6fnezbZH+AnI2J9ROyqblBk1yx2A7OBPwFWAw9JekHaXxMRTxXo4xURsT0ifk8WuGYPUW4fcCJZQPxjRPyfOs/5LuDqiLg99eVa4AlgbkWZL0XEtvS6nwTOq3idWr9XawMOHHYomET2ib7fg+kYwOVAD3CzpC2SlgJERA/ZyOIy4BFJKyT11zkRuDGlYR4j+4T/JPAc4B/JgsCKlBb7jKQjhmjXrcCfAmem7Z+SBY0/SftF/LZiew/wzCHKfZBsRHWHpI2S3l7nOU8E/qa/n6mvUxn43UE2KuvX8Pdq7cGBww4F28n+CPY7IR0jInZHxN9ExAzgvwHv77+WERHfiohXproBfDrV3wbMj4hjKh5Pi4iH0qfrj0bELOC/AucA5w/Rrv7AcUbavpXGgeOgPrVHxG8j4l0RMYls5PTlOjOptgGfrOrnkRFxQ0WZqRXbuX6vduhz4LBDwQ3ARyR1SjoOuBS4Hv7z4u/JkgTsIhs5PCnp+ZJelS6i/xH4f+kcZLn7T0o6MT1Hp6SFafssSS+R1JGeb19FvWq3AmcBT4+IXmANMI8s1fUfQ9R5mOwayLBIeqOkKWn3D2SBqL991c/9FWCxpNOVeYakP5N0VEWZiyVNSZMD/hfQf7G95u91uO22scWBww4FnwC6gQ3APcCd6RjATOBHwOPAL4AvR8RPya5vfAp4lCwNdDzZH0aALwArydIwu8kulJ+ezj0X+CeyP5b3kQWH62s1KiLuT6+7Ju3vArYAP4uIof7Ifo3sustjkv61wO+g38uA2yU9nvrw3oj4dTp3GXBteu43RUQ32XWOL5EFmR7gbVXP9y3g5tTuLTT+vVobkK9nmVktkrYC74yIHzW7LdZaPOIwM7NCHDjMzKwQp6rMzKwQjzjMzKyQw5vdgNFw3HHHxbRp05rdDDOzMWX9+vWPRkRn9fG2CBzTpk2ju7u72c0wMxtTJD1Y67hTVWZmVogDh5mZFeLAYWZmhThwmJlZIQ4cZmZWiAOHmZkV4sBhZmaFOHDUERFs3L4TL8tiZjbAgaOOTX27ePf1d7Kp74CvlDYza1sOHHXMmjiBq946h1kTJzS7KWZmLaMtlhwZLkm8aNLRzW6GmVlL8YjDzMwKceAwM7NCHDjq8KwqM7MDOXDU4VlVZmYHcuCow7OqzMwO5MBhZmaFOHDU4VSVmdmBHDjqcKrKzOxAvgGwDt8AaGZ2II84zMysEAcOMzMrxIGjDt8AaGZ2oFIDh6R5kjZL6pG0tMZ5Sboind8gaU6eupLek85tlPSZstrvWVVmZgcq7eK4pA7gSuA1QC+wTtLKiNhUUWw+MDM9TgeuAk6vV1fSWcBC4JSIeELS8WX1wbOqzMwOVOaI4zSgJyK2RMReYAXZH/xKC4HrIrMWOEbSxAZ13w18KiKeAIiIR8rqQP+sKkllvYSZ2ZhTZuCYDGyr2O9Nx/KUqVf3ecAZkm6XdKukl9V6cUmLJHVL6t6xY8dBdMPMzCqVGThqfUyvvso8VJl6dQ8HngXMBf4n8B3VGBJExDUR0RURXZ2dnflbbWZmdZUZOHqBqRX7U4DtOcvUq9sL/EtKb90BPAUcN4LtHsQzq8zMBiszcKwDZkqaLmkccC6wsqrMSuD8NLtqLrAzIvoa1P1X4FUAkp4HjAMeLasTnlllZjZYabOqImK/pCXAaqADWB4RGyUtTueXAauABUAPsAe4sF7d9NTLgeWS7gX2AhdEicMBz6wyMxtM7ZCC6erqiu7u7mHVjQg29e1i1sQJnl1lZm1F0vqI6Ko+7jvHG3CqysxsMAeOBpyqMjMbzMuqN+Cl1c3MBvOIw8zMCnHgMDOzQhw4GvANgGZmgzlwNOBZVWZmgzlwNOBZVWZmgzlwmJlZIQ4cDThVZWY2mANHA05VmZkN5hsAG/ANgGZmg3nEYWZmhThwmJlZIQ4cDfgGQDOzwRw4GvCsKjOzwRw4GvCsKjOzwTyrqgHPqjIzG8wjDjMzK8SBw8zMCnHgaMCzqszMBis1cEiaJ2mzpB5JS2ucl6Qr0vkNkuY0qivpMkkPSborPRaU2QfPqjIzG6y0wCGpA7gSmA/MAs6TNKuq2HxgZnosAq7KWfdzETE7PVaV1QfwrCozs2pljjhOA3oiYktE7AVWAAuryiwErovMWuAYSRNz1jUzsyYoM3BMBrZV7PemY3nKNKq7JKW2lkt6Vq0Xl7RIUrek7h07dgy3D05VmZlVKTNwqMax6ivMQ5WpV/cq4CRgNtAH/EOtF4+IayKiKyK6Ojs7czW4FqeqzMwGK/MGwF5gasX+FGB7zjLjhqobEQ/3H5T0FeCmkWuymZk1UuaIYx0wU9J0SeOAc4GVVWVWAuen2VVzgZ0R0VevbroG0u8NwL0l9sGpKjOzKqWNOCJiv6QlwGqgA1geERslLU7nlwGrgAVAD7AHuLBe3fTUn5E0myx1tRW4qKw+gFNVZmbV1A43tnV1dUV3d3ezm2FmNqZIWh8RXdXHfee4mZkV4sBhZmaFOHA04LWqzMwGc+BowLOqzMwGc+BowLOqzMwGc+AwM7NCHDgacKrKzGwwB44GnKoyMxuszLWqDgmSeNGko5vdDDOzluERh5mZFeLAYWZmhThw5OCbAM3MBjhw5OCZVWZmAxw4cvDMKjOzAQ4cZmZWiANHDk5VmZkNcODIwakqM7MBDhxmZlaIA0cOTlWZmQ1w4MjBqSozswFeqyoHr1dlZjag1BGHpHmSNkvqkbS0xnlJuiKd3yBpToG6H5AUko4rsw9mZjZYaYFDUgdwJTAfmAWcJ2lWVbH5wMz0WARclaeupKnAa4DflNV+MzOrrcwRx2lAT0RsiYi9wApgYVWZhcB1kVkLHCNpYo66nwM+CIzK4lFeq8rMbECZgWMysK1ivzcdy1NmyLqSXg88FBF313txSYskdUvq3rFjx/B6kHhWlZnZgDIDh2ocq/7IPlSZmsclHQlcAlza6MUj4pqI6IqIrs7OzoaNrcezqszMBpQZOHqBqRX7U4DtOcsMdfwkYDpwt6St6fidkp47oi03M7MhlRk41gEzJU2XNA44F1hZVWYlcH6aXTUX2BkRfUPVjYh7IuL4iJgWEdPIAsyciPhtif1wqsrMrEJp93FExH5JS4DVQAewPCI2Slqczi8DVgELgB5gD3BhvbpltbURp6rMzAaUegNgRKwiCw6Vx5ZVbAdwcd66NcpMO/hWmplZEV5yJAenqszMBjhw5OBUlZnZgMKBQ9KzJJ1SRmPMzKz15Qockn4qaYKkY4G7ga9L+my5TWsdTlWZmQ3IO+I4OiJ2AX8OfD0iXgqcXV6zWotTVWZmA/IGjsPTGlJvAm4qsT0tqX9ZdanWDe1mZu0lb+D4GNk9FQ9ExDpJM4BfldcsMzNrVbnu44iI7wLfrdjfAvxFWY0yM7PWlffi+PMk/VjSvWn/FEkfKbdprcPLqpuZDcibqvoK8GFgH0BEbCBbP6oteFaVmdmAvIHjyIi4o+rY/pFuTKvyrCozswF5A8ejkk4ifZ+GpL8E+kprlZmZtay8geNi4GrgBZIeAt4HvLusRrUap6rMzAbknVW1BThb0jOAwyJid7nNai1OVZmZDcg7q+q9kiaQfWfG5yTdKem15TatdUhi1sQJbOrb5ZlVZtb28qaq3p6WHHktcDzZFy59qrRWtSCnq8zMMnkDR/9aGwvI1qq6u+JYW3C6yswskzdwrJd0M1ngWC3pKOCp8pplZmatKm/geAewFHhZROwBjiB9P3i7cKrKzCyTN3C8HNgcEY9JeivwEWBnec1qPU5VmZll8gaOq4A9kk4FPgg8CFxXWqvMzKxl5Q0c+yObh7oQ+EJEfAE4qlElSfMkbZbUI2lpjfOSdEU6v0HSnEZ1JX08lb1L0s2SJuXsw0FxqsrMLJM3cOyW9GHgr4DvS+ogu84xpFTmSmA+MAs4T9KsqmLzgZnpsYhsZNOo7uURcUpEzCb7UqlLc/bhoDhVZWaWyRs43gw8QXY/x2+BycDlDeqcBvRExJaI2AusIBuxVFoIXBeZtcAx6ZsGh6yb7ifp9wzS+llmZjY6cgWOFCy+CRwt6RzgjxHR6BrHZGBbxX5vOpanTN26kj4paRvwFoYYcUhaJKlbUveOHTsaNLUxp6rMzDJ5lxx5E3AH8Eay7x2/Pa2QW7dajWPVo4OhytStGxGXRMRUsmC2pNaLR8Q1EdEVEV2dnZ0NmtqYU1VmZplcixwCl5Ddw/EIgKRO4EfAP9Wp0wtMrdifAmzPWWZcjroA3wK+D/xt4y6YmdlIyHuN47D+oJH8LkfddcBMSdMljSP7xsCVVWVWAuen2VVzgZ0R0VevrqSZFfVfD/wyZx8OilNVZmaZvCOOH0paDdyQ9t8MrKpXISL2S1oCrAY6gOURsVHS4nR+WXqOBUAP2cq7F9arm576U5KeT7bkyYPA4px9OChOVZmZZZR3mXBJfwG8guz6w20RcWOZDRtJXV1d0d3dfVDPERFs6tvFrIkTkNpqfUcza1OS1kdEV/XxvKkqIuKfI+L9EfHXYylojBSnqszMMnVTVZJ2U/s+CQEREW2Tt3GqyswsUzdwRETDZUXMzKy95E5VtTunqszMMg4cOTlVZWaWceAwM7NCHDhycqrKzCzjwJGTU1VmZhkHDjMzK8SBIyenqszMMg4cOTlVZWaWceAwM7NCHDhycqrKzCzjwJGTU1VmZhkHjpwkMWviBDb17SLvUvRmZociB44CnK4yM3PgKMTpKjMzBw4zMyvIgaMAp6rMzBw4CnGqyszMgaOQiOCBHY97VpWZtbVSA4ekeZI2S+qRtLTGeUm6Ip3fIGlOo7qSLpf0y1T+RknHlNmHSjfd08f7v303N93TN1ovaWbWckoLHJI6gCuB+cAs4DxJs6qKzQdmpsci4KocdW8BXhwRpwD3Ax8uqw/VznnJRD775lM55yUTR+slzcxaTpkjjtOAnojYEhF7gRXAwqoyC4HrIrMWOEbSxHp1I+LmiNif6q8FppTYh0EkcVLnM5E0Wi9pZtZyygwck4FtFfu96VieMnnqArwd+EGtF5e0SFK3pO4dO3YUbHptnlVlZlZu4Kj1sbz6qvJQZRrWlXQJsB/4Zq0Xj4hrIqIrIro6OztzNLcxz6oyM4PDS3zuXmBqxf4UYHvOMuPq1ZV0AXAO8OrwFCczs1FV5ohjHTBT0nRJ44BzgZVVZVYC56fZVXOBnRHRV6+upHnAh4DXR8SeEtt/AKeqzMxKHHFExH5JS4DVQAewPCI2Slqczi8DVgELgB5gD3Bhvbrpqb8EjAduSRep10bE4rL6UcmpKjOzclNVRMQqsuBQeWxZxXYAF+etm46fPMLNzK3/BsAXPvcoz6wys7blO8cL8A2AZmYljzgONf03/vkGQDNrZx5xFOAbAM3MHDgK8awqMzMHjkI8q8rMzIHDzMwKcuAowKkqMzMHjkKcqjIzc+AoRBIvfO5RfG/Ddp566qlmN8fMrCkcOAryTYBm1u58A2BBvgnQzNqdRxxmZlaIA0dBTlWZWbtzqqogp6rMrN15xGFmZoU4cBTkVJWZtTunqgpyqsrM2p1HHAV5aXUza3cOHAV5vSoza3cOHAV5vSoza3cOHAVFBA/seJyIaHZTzMyaotTAIWmepM2SeiQtrXFekq5I5zdImtOorqQ3Stoo6SlJXWW2vxbPqjKzdlfarCpJHcCVwGuAXmCdpJURsami2HxgZnqcDlwFnN6g7r3AnwNXl9X2ejyryszaXZkjjtOAnojYEhF7gRXAwqoyC4HrIrMWOEbSxHp1I+K+iNhcYrvrksT0Zx/JTRv6vLS6mbWlMu/jmAxsq9jvJRtVNCozOWfduiQtAhYBnHDCCUWq1rWpbxdv+3o3j+3ZB4LXnzp5xJ7bzGwsKDNw1LrRofqK8lBl8tStKyKuAa4B6OrqGrEr2bMmTuAbF3bx60f3OF1lZm2pzFRVLzC1Yn8KsD1nmTx1m0ISL5p0dO3QZmbWBsoMHOuAmZKmSxoHnAusrCqzEjg/za6aC+yMiL6cdZvGM6vMrJ2VlqqKiP2SlgCrgQ5geURslLQ4nV8GrAIWAD3AHuDCenUBJL0B+CLQCXxf0l0R8bqy+lGLZ1aZWTsrdZHDiFhFFhwqjy2r2A7g4rx10/EbgRtHtqXFRAS9f9jjmwDNrC35zvFhuHrNFj7zw/u5es2WZjfFzGzUeVn1YbjojBmDfpqZtROPOMzMrBAHjmFwqsrM2plTVcPgVJWZtTOPOIaho6ODi86YwdVrtvDkk082uzlmZqPKgWOYnK4ys3blVNUwOV1lZu3KI45higge3vVH3wRoZm3HgWOYPr7qPq79+W/4+Kr7mt0UM7NR5VTVMP3vBS8c9NPMrF14xDFMkjj+qPHc17fb6SozaysOHMN09ZotXL76V7x1+To29e1qdnPMzEaNU1XDdNEZM4gIzjy5k1kTJzS7OWZmo8YjjmHq6OjgXa+YxuW3bGbfvn3Nbo6Z2ahx4DgI77z+Tm67/3e88/o7m90UM7NR48BxEL761jmc+bxn89W3zml2U8zMRo0Dx0Ho6OhgzpSjeMd1d7J3795mN8fMbFQ4cByEq9ds4fP/vpU1PU5XmVn78Kyqg3DRGTPYu3cv63+z2+kqM2sbHnEchI6ODt7z6hdw6qSn88pP/4Qv3Hyfl1k3s0NeqYFD0jxJmyX1SFpa47wkXZHOb5A0p1FdScdKukXSr9LPZ5XZh0auXrOFL93WyyP/dz+f+/ctXmbdzA55pQUOSR3AlcB8YBZwnqRZVcXmAzPTYxFwVY66S4EfR8RM4Mdpv2kuOmMGS86cwvHPOJy/ftUML7NuZoe8Mq9xnAb0RMQWAEkrgIXApooyC4HrIlvsaa2kYyRNBKbVqbsQ+NNU/1rgp8CHSuxHXR0dHXxgwam859UvYOEXb+Vrtw6MOCQ4ogP27j+wns/5nM+13rlWacdInXv604/gy//jVOZMPx5JBxYepjIDx2RgW8V+L3B6jjKTG9R9TkT0AUREn6Tja724pEVkoxhOOOGEYXYhv/O+to5fPlrjDvIab6zP+ZzPtfC5VmnHCJzb9fg+3vTVblYueSUvmnR0nQrFlBk4aoW36mVkhyqTp25dEXENcA1AV1dX6cvX3vCOl7Hwi7ey/Q8DwaNVP4X4nM/5XHuNOEZ6Pb0yA0cvMLVifwqwPWeZcXXqPixpYhptTAQeGdFWD9P48eP54Qde2+xmmJmVrsxZVeuAmZKmSxoHnAusrCqzEjg/za6aC+xMaah6dVcCF6TtC4B/K7EPZmZWpbQRR0Tsl7QEWA10AMsjYqOkxen8MmAVsADoAfYAF9arm576U8B3JL0D+A3wxrL6YGZmB1I7fHtdV1dXdHd3N7sZZmZjiqT1EdFVfdx3jpuZWSEOHGZmVogDh5mZFeLAYWZmhbTFxXFJO4AHh1n9OODREWxOM7kvredQ6Qe4L63qYPpyYkR0Vh9si8BxMCR115pVMBa5L63nUOkHuC+tqoy+OFVlZmaFOHCYmVkhDhyNXdPsBowg96X1HCr9APelVY14X3yNw8zMCvGIw8zMCnHgMDOzQhw46pA0T9JmST2Smvrd5nlI2irpHkl3SepOx46VdIukX6Wfz6oo/+HUt82SXte8loOk5ZIekXRvxbHCbZf00vQ76JF0hUby+zIPri+XSXoovTd3SVrQ6n2RNFXSTyTdJ2mjpPem42PufanTl7H4vjxN0h2S7k59+Wg6PnrvS0T4UeNBtpz7A8AMsi+WuhuY1ex2NWjzVuC4qmOfAZam7aXAp9P2rNSn8cD01NeOJrb9TGAOcO/BtB24A3g52bdI/gCY3yJ9uQz4QI2yLdsXYCIwJ20fBdyf2jvm3pc6fRmL74uAZ6btI4Dbgbmj+b54xDG004CeiNgSEXuBFcDCJrdpOBYC16bta4H/XnF8RUQ8ERG/JvtOlNNGv3mZiLgN+H3V4UJtV/aNkBMi4heR/a+4rqLOqBmiL0Np2b5ERF9E3Jm2dwP3AZMZg+9Lnb4MpZX7EhHxeNo9Ij2CUXxfHDiGNhnYVrHfS/1/aK0ggJslrZe0KB17TmTfqkj6eXw6Phb6V7Ttk9N29fFWsUTShpTK6k8jjIm+SJoG/BeyT7dj+n2p6guMwfdFUoeku8i+OvuWiBjV98WBY2i1cn2tPnf5FRExB5gPXCzpzDplx2L/+g3V9lbu01XAScBsoA/4h3S85fsi6ZnAPwPvi4hd9YrWONbqfRmT70tEPBkRs4EpZKOHF9cpPuJ9ceAYWi8wtWJ/CrC9SW3JJSK2p5+PADeSpZ4eTkNS0s9HUvGx0L+ibe9N29XHmy4iHk7/2Z8CvsJAWrCl+yLpCLI/tN+MiH9Jh8fk+1KrL2P1fekXEY8BPwXmMYrviwPH0NYBMyVNlzQOOBdY2eQ2DUnSMyQd1b8NvBa4l6zNF6RiFwD/lrZXAudKGi9pOjCT7EJZKynU9jQ83y1pbpodcn5Fnabq/w+dvIHsvYEW7kt63a8B90XEZytOjbn3Zai+jNH3pVPSMWn76cDZwC8ZzfdlNGcDjLUHsIBs9sUDwCXNbk+Dts4gmzlxN7Cxv73As4EfA79KP4+tqHNJ6ttmmjD7qKr9N5ClCvaRfRJ6x3DaDnSR/ed/APgSaXWEFujLPwL3ABvSf+SJrd4X4JVkqYsNwF3psWAsvi91+jIW35dTgP9Ibb4XuDQdH7X3xUuOmJlZIU5VmZlZIQ4cZmZWiAOHmZkV4sBhZmaFOHCYmVkhDhxmJZP0PklHNrsdZiPF03HNSiZpK9AVEY82uy1mI8EjDrMRlO7g/376roR7Jf0tMAn4iaSfpDKvlfQLSXdK+m5aP6n/+1Q+nb5r4Q5JJ6fjb0zPdbek25rXO7OMA4fZyJoHbI+IUyPixcDnydb/OSsizpJ0HPAR4OzIFqTsBt5fUX9XRJxGdhfv59OxS4HXRcSpwOtHpxtmQ3PgMBtZ9wBnp5HDGRGxs+r8XLIv1vlZWhb7AuDEivM3VPx8edr+GfANSe8i+4Ixs6Y6vNkNMDuURMT9kl5Ktg7S30m6uaqIyL4/4byhnqJ6OyIWSzod+DPgLkmzI+J3I912s7w84jAbQZImAXsi4nrg78m+QnY32deVAqwFXlFx/eJISc+reIo3V/z8RSpzUkTcHhGXAo8yeIlss1HnEYfZyHoJcLmkp8hWx303WcrpB5L60nWOtwE3SBqf6nyEbBVmgPGSbif7UNc/Krlc0kyy0cqPyVZANmsaT8c1axGetmtjhVNVZmZWiEccZmZWiEccZmZWiAOHmZkV4sBhZmaFOHCYmVkhDhxmZlbI/wfmfNbQms87KAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_scatter(list(range(3001)), losses, 'steps', 'losses')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "029c05cd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEWCAYAAACT7WsrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAgFElEQVR4nO3de5RdZZnn8e8vlQSEEIKm0MoFSDCNlCAQq2NoFO+Yioxxpsc2WSKX0clEySh2aw9qLy9rtdqK7Sg9TDIR6CaNGhHRycJIcBRBWVxSwZCQhGgRQIoUpAKSBNIQKnnmj/1W59Spc6pOXXadqjq/D+usc/Z+3733s3Ooeuq97L0VEZiZmZUyrtoBmJnZyOUkYWZmZTlJmJlZWU4SZmZWlpOEmZmV5SRhZmZlOUnYmCPp55IuGeq6Q0HS5yRd20v5pZJ+O1zxmPVFvk7CLCPpUuCjEfHmYTreKcCjwISI6BzKGCR9CXhtRFw0yDCtxrklYWZmZTlJ2Igl6TOSfly07p8kfbuP7X4t6aPp86WSfivpm5L+JOlRSc3FdSWdDqwEzpX0vKTnUvlCSdsk7Zf0pKRPlznm45LemD5fJCkkNablj0r6afr8JUk3ps3uSu/PpWOeW7C/kvGWOO7/SHHtl7RD0jslLQA+B3ww7ffBVPd4SddJak/b/L2kuoJ/p7vTv+9eSQ9LemfBcS6VtDMd51FJH+rtO7Cxw0nCRrIbgQWSpgBIGg98EPjXfu7nTcAOYCrwDeA6SSqsEBHbgWXAPRExKSKmpKLrgP8WEccBZwC/KnOMO4G3pc/nAzuBtxYs31lim/PT+5R0zHsqjRdA0mnAcuDPU3zvAR6LiNuArwI/TPs9K21yA9AJvBY4B7gA+GjBLt+U4p4KfBG4RdIrJR0LXA00p+P8BbCpzL+DjTFOEjZiRUQ72V/bH0irFgB7ImJjP3f1eER8NyIOkf2ibABeXeG2LwONkiZHxJ8i4oEy9e7kSFJ4C/C1guW3UjpJDDbeQ8BRKb4JEfFYRDxSaoeSXg00A1dExAsRsRv4n8Digmq7gW9HxMsR8UOyRPXeVHYYOEPSKyKiPSK29uN8bBRzkrCR7gaga/D1IvrfigB4qutDRBxIHydVuO1fAguBxyXdWdglVORO4C2SXgPUAT8EzkuD08fTv7+8K4o3IlqBK4AvAbslrZE0rcw+TwYmAO2Snkvdaf8HOLGgzpPRfSbL48C0iHiBrAW3LG3/M0mv68f52CjmJGEj3U+BN0g6A7gQ+F6Ox+ox1S8iNkTEIrJfpj8Fbiq5YfYL+wDwCeCuiNhP9st+KfDbiDhcyfH6HXDE99NMqJPT/r5eZt9PAC8BUyNiSnpNjojXF9SZXtStdRKwKx1nfUS8m6xV8zDw3cHGbqODk4SNaBHxInAz8H3g/oj4Y46HexqYIWkigKSJkj4k6fiIeBnYR9bFU86dZGMEXV1Lvy5aLtZB1o0zeyDBSjpN0jskHQW8CPxbQXxPA6dIGgf/3nV3O/CPkiZLGifpVElvLdjlicAnJE2Q9AHgdGCdpFdLel8am3gJeJ7e/x1sDHGSsNHgBuBMBtbV1B+/ArYCT0nak9Z9GHhM0j6y7pberju4EziOI7OWipe7SV1JXwHuTl1A8/sZ71HAPwB7yFotJ5LNagL4UXp/RlLXOMrFwERgG/AnsuTbULC/+4A5aX9fAf5zRDxD9nvib8haFc+SjbF8vJ+x2ijli+lsxJN0ElkXx2siYl+14xmLhvtCQhs93JKwES11l/w1sMYJwmz4ja92AGblpD7wp8lm2SwoKnu+zGbNEfGbvGMzqxW5dzelqz+/QzYt8NqI+IeicqXyhWSzQy7tmosu6VNkF/sEsAW4LA1kmpnZMMi1uyld8n8N2UU8jcCSrlsVFGgmGyybQzZdcEXadjrZdMKmiDiDLMksxszMhk3e3U3zgNaI2AkgaQ2wiGx2RZdFwOp0Ec+9kqZI6ppxMR54haSXgWNIc7bLmTp1apxyyilDfApmZmPbxo0b90REfamyvJPEdLKLeLq0kd0fpq860yOiRdI3gT+Szf++PSJuLz6ApKVkLRBOOukkWlpahjB8M7OxT9Lj5crynt3U46Zk9LwStGQdSSeQtTJmAdOAYyX1mKMeEasioikimurrSyZCMzMboLyTRBsws2B5Bj27jMrVeRfwaER0pKtdbyG7+6SZmQ2TvJPEBmCOpFnpVgeLgbVFddYCFyszH9ibbiHwR2C+pGPSDKh3AttzjtfMzArkOiYREZ2SlgPryWYnXR8RWyUtS+UrgXVk01+7bpB2WSq7T9LNwANk98D/HbAqz3jNzKy7MXVbjqampvDAtZlZ/0jaGBFNpcp8Ww4zMyvLScLMzMpykkgigq279jKWut/MzAbLSSLZ1r6Pj934ANvafaNRM7MuThJJY8NkVlw0l8aGydUOxcxsxHCSMDOzspwkEnc3mZn15CSRnP6a4/j0e/6M019zXLVDMTMbMZwkku1P7eeb63/P9qf2VzsUM7MRw0ki8cC1mVlPThJmZlaWk0TigWszs56cJBJ3N5mZ9eQkYWZmZTlJJO5uMjPryUki8XUSZmY9OUkkvk7CzKyn3JOEpAWSdkhqlXRliXJJujqVb5Y0N60/TdKmgtc+SVfkFadbEmZmPeWaJCTVAdcAzUAjsERSY1G1ZmBOei0FVgBExI6IODsizgbeSPb865/kFatbEmZmPeXdkpgHtEbEzog4CKwBFhXVWQSsjsy9wBRJDUV13gk8EhGP5xWop8CamfWUd5KYDjxRsNyW1vW3zmLgB6UOIGmppBZJLR0dHYMM18zMCuWdJFRiXfHzQXutI2ki8D7gR6UOEBGrIqIpIprq6+sHHKinwJqZ9ZR3kmgDZhYszwB29bNOM/BARDydS4SJu5vMzHrKO0lsAOZImpVaBIuBtUV11gIXp1lO84G9EdFeUL6EMl1NQ0kSjQ2T2da+j4jixo6ZWW3KNUlERCewHFgPbAduioitkpZJWpaqrQN2Aq3Ad4GPd20v6Rjg3cAtecbZxV1OZmbdaSz91dzU1BQtLS0D3v7w4cPcuqWdC89sYNw4X2doZrVB0saIaCpV5t+EBXythJlZd04SBTx4bWbWnZOEmZmV5SRRwAPXZmbdOUkU8E3+zMy6c5Io4IFrM7PunCQKeODazKw7JwkzMyvLSaKAB67NzLpzkijggWszs+6cJAp44NrMrDsniQIeuDYz685JwszMynKSKOCBazOz7pwkCnjg2sysOyeJAh64NjPrzkmigAeuzcy6yz1JSFogaYekVklXliiXpKtT+WZJcwvKpki6WdLDkrZLOjfveM3M7Ihck4SkOuAaoBloBJZIaiyq1gzMSa+lwIqCsu8At0XE64CzyJ6TnRsPXJuZdZd3S2Ie0BoROyPiILAGWFRUZxGwOjL3AlMkNUiaDJwPXAcQEQcj4rk8g/XAtZlZd3knienAEwXLbWldJXVmAx3AP0v6naRrJR1bfABJSyW1SGrp6OgYVLAeuDYz6y7vJKES66LCOuOBucCKiDgHeAHoMaYREasioikimurr6wcVrAeuzcy6yztJtAEzC5ZnALsqrNMGtEXEfWn9zWRJIzeSaGyYzLb2fUQU5zIzs9qTd5LYAMyRNEvSRGAxsLaozlrg4jTLaT6wNyLaI+Ip4AlJp6V67wS25RyvB6/NzAqMz3PnEdEpaTmwHqgDro+IrZKWpfKVwDpgIdAKHAAuK9jFfwe+lxLMzqKyXHjw2szsiFyTBEBErCNLBIXrVhZ8DuDyMttuApryjK9Y1+D1qfWTeP2044fz0GZmI46vuC7iwWszsyOcJMzMrCwniSIeuDYzO8JJoogHrs3MjnCSKOKrrs3MjnCSKOKBazOzI5wkzMysLCeJIh64NjM7wkmiiAeuzcyOcJIo4oFrM7MjnCSKuCVhZnaEk0QRtyTMzI5wkijiKbBmZkc4SZiZWVlOEkU8BdbM7AgniSIeuDYzO8JJoogHrs3Mjsg9SUhaIGmHpFZJV5Yol6SrU/lmSXMLyh6TtEXSJkkteccKbkmYmRXKNUlIqgOuAZqBRmCJpMaias3AnPRaCqwoKn97RJwdEcPyGFO3JMzMjsi7JTEPaI2InRFxEFgDLCqqswhYHZl7gSmSGnKOqyxPgTUzOyLvJDEdeKJguS2tq7ROALdL2ihpaakDSFoqqUVSS0dHx6ADlkRjw2S2te8jIga9PzOz0SzvJKES64p/8/ZW57yImEvWJXW5pPN7VIxYFRFNEdFUX18/uGgTT4M1M8vknSTagJkFyzOAXZXWiYiu993AT8i6r3LnwWszs0zeSWIDMEfSLEkTgcXA2qI6a4GL0yyn+cDeiGiXdKyk4wAkHQtcADyUc7yAB6/NzLqMz3PnEdEpaTmwHqgDro+IrZKWpfKVwDpgIdAKHAAuS5u/GviJpK44vx8Rt+UZbxe3JMzMMrkmCYCIWEeWCArXrSz4HMDlJbbbCZyVd3yldLUkTq2fxOunHV+NEMzMRgRfcV2CWxJmZhkniRI8JmFmlnGSKMEX1JmZZZwkzMysLCeJEnwxnZlZxkmiBA9cm5llnCRK8MC1mVnGSaIEtyTMzDJOEiW4JWFmlnGSKMFTYM3MMk4SZmZWVkVJQtJ56U6sSLpI0rcknZxvaNXjKbBmZplKWxIrgAOSzgL+FngcWJ1bVFXmgWszs0ylSaIz3a11EfCdiPgOMGZ/g3rg2swsU+mtwvdL+ixwEXC+pDpgQn5hVZdbEmZmmUpbEh8EXgI+EhFPAdOBq3KLqsrckjAzy1SaJD4VEd+KiN8ARMQfgddXsqGkBZJ2SGqVdGWJckm6OpVvljS3qLxO0u8k3VphrIPW2DCZ//2hcyAg62UzM6tNlSaJd5dY19zXRqlb6ppUtxFYIqmxxH7mpNdSskHyQp8EtlcY55CQhCQ+9j3PcDKz2tZrkpD0MUlbgNPSX/ldr0eBLRXsfx7QGhE7I+IgsIZs8LvQImB1ZO4FpkhqSMefAbwXuLaf5zVoHpcwM+t74Pr7wM+BrwGFXUX7I+LZCvY/HXiiYLkNeFMFdaYD7cC3yabcDvtvaj/n2sys7+6miIjHgMuB/QUvJL2ygv2r1D4rqSPpQmB3RGzs9QDSUkktklo6OjoqCKkybkmYmfWdJL6f3jcCLel9Y8FyX9qAmQXLM4BdFdY5D3ifpMfIuqneIenG4gNExKqIaIqIpvr6+gpCqoxnOJmZ9ZEkIuLC9D4rIman967X7Ar2vwGYI2mWpInAYmBtUZ21wMVpltN8YG9EtEfEZyNiRkSckrb7VURc1N8THCi3JMzMKr+YDknTgZMLt4mIu3rbJiI6JS0H1gN1wPURsVXSslS+ElgHLARagQPAZf09iTx4TMLMDFTJdQCSvk52Qd024FBaHRHxvhxj67empqZoaamkF6xvhw8f5tYt7Vx4ZgPjxvlmuWY2dknaGBFNpcoqbUm8HzgtIl4asqhGOLckzMwqv5huJ2P4Xk2leEzCzKzylsQBYJOkX5LdwwmAiPhELlGNAG5JmJlVniTW0nNW0pjmloSZWYVJIiJuyDuQkcYtCTOzPpKEpJsi4q/S/Zt6TIOKiDfkFlmVuSVhZtZ3S+KT6f3CvAMZadySMDPrI0lERHt6f7y3epLuiYhzhzKwanNLwsys8imwfTl6iPYzYvjeTWZmQ5ckxtzj29ySMDMbuiQx5rglYWZWYZKQtFzSCb1VGaJ4Rgw/59rMrPKWxGuADZJukrRAUnFS+PAQx1V1fs61mVmFSSIi/g6YA1wHXAr8QdJXJZ2ayh/KLcIq8riEmdW6isckIutzeSq9OoETgJslfSOn2KpuW/s+vvqzh92SMLOaVdFtOSR9ArgE2ANcC3wmIl6WNA74A/C3+YVYPUpDLRp7Qy5mZhWp9AZ/U4H/VHxRXUQcljRmr8Y+veE4Pvfe13F6g7ubzKw2VTom8YVyV11HxPbetk0D3TsktUq6skS5JF2dyjdLmpvWHy3pfkkPStoq6cuVxDqUPA3WzGpdrtdJSKoDrgGagUZgiaTGomrNZIPic4ClwIq0/iXgHRFxFnA2sEDS/DzjLeaBazOrdXlfTDcPaI2InRFxEFgDLCqqswhYHZl7gSmSGtLy86nOhPQa1gsW3JIws1qXd5KYDjxRsNyW1lVUR1KdpE3AbuAXEXFf8QEkLZXUIqmlo6NjKGN3S8LMal7eSaLUtKDi1kDZOhFxKCLOBmYA8ySd0aNixKqIaIqIpvr6+sHG241bEmZW6/JOEm3AzILlGcCu/taJiOeAXwMLhjzCXrglYWa1Lu8ksQGYI2mWpInAYno+K3stcHGa5TQf2BsR7ZLqJU0BkPQK4F3AwznH241bEmZW6yq9TmJAIqJT0nJgPVAHXB8RWyUtS+UrgXXAQqAVOABcljZvAG5IM6TGATdFxK15xlvMLQkzq3W5JgmAiFhHlggK160s+BzA5SW22wyck3d8vem6LcfsqcdyxvQp1QzFzKwq/DyJXvi2HGZW65wkeuHbcphZrXOS6MX2p/Zz1W07uHVLux88ZGY1yUmiF40Nk/nMgtO4av0O3y7czGpS7gPXo5kkLjyzAcAznMysJrkl0QdfK2FmtcxJog++VsLMapmTRB/8CFMzq2VOEn3wtRJmVsucJPrgayXMrJY5SfTB3U1mVsucJPrg7iYzq2VOEn1wd5OZ1TIniT64u8nMapmTRB/c3WRmtcxJog/ubjKzWuYk0Qd3N5lZLcs9SUhaIGmHpFZJV5Yol6SrU/lmSXPT+pmS7pC0XdJWSZ/MO9ZS3N1kZrUs1ySRnk99DdAMNAJLJDUWVWsG5qTXUmBFWt8J/E1EnA7MBy4vsW3uGqdN5tpL3gjgZ0qYWc3JuyUxD2iNiJ0RcRBYAywqqrMIWB2Ze4Epkhoioj0iHgCIiP3AdmB6zvH2IGUtiI/c0MLWXXuH+/BmZlWVd5KYDjxRsNxGz1/0fdaRdApwDnBf8QEkLZXUIqmlo6NjKGLuwV1OZlar8k4SpX6rFvfZ9FpH0iTgx8AVEdFj9DgiVkVEU0Q01dfXDyrYcjzDycxqVd5Jog2YWbA8A9hVaR1JE8gSxPci4pYc4+yVZziZWa3KO0lsAOZImiVpIrAYWFtUZy1wcZrlNB/YGxHtygYDrgO2R8S3co6zdwEvHzrcsw1kZjbG5fqM64jolLQcWA/UAddHxFZJy1L5SmAdsBBoBQ4Al6XNzwM+DGyRtCmt+1xErMszZjMzOyLXJAGQfqmvK1q3suBzAJeX2O63lB6vGHaSmFA37t9nOpmZ1QpfcV0BD1ybWa1ykqiAB67NrFY5SVTCA9dmVqOcJCogifHjxM49L/jWHGZWU5wkKtA4bTKfe+/r+Mq67b41h5nVFCeJCkjitfXHofSfmVmtcJKokGc4mVktcpKokGc4mVktcpKolGc4mVkNcpIwM7OynCTMzKwsJwkzMyvLScLMzMpykjAzs7KcJCrkW3OYWS1ykqiQb81hZrXISaJCkjh16iQ6D4WvlTCzmpF7kpC0QNIOSa2SrixRLklXp/LNkuYWlF0vabekh/KO08zMeso1SUiqA64BmoFGYImkxqJqzcCc9FoKrCgo+xdgQZ4xmplZeXm3JOYBrRGxMyIOAmuARUV1FgGrI3MvMEVSA0BE3AU8m3OMFYsIOg8d9sC1mdWMvJPEdOCJguW2tK6/dcqStFRSi6SWjo6OAQdaiZ17XmD/i53s3PNCrscxMxsp8k4SpR6+UPxneCV1yoqIVRHRFBFN9fX1/Qquv2ZPPZbjjh7P7KnH5nocM7ORIu8k0QbMLFieAewaQJ0RQRJ148SjvlbCzGpE3kliAzBH0ixJE4HFwNqiOmuBi9Msp/nA3ohozzmuAZGyRs+Xb/W1EmZWG3JNEhHRCSwH1gPbgZsiYqukZZKWpWrrgJ1AK/Bd4ONd20v6AXAPcJqkNkkfyTPevjROm8wX/8Pp2YIbEmZWA8bnfYCIWEeWCArXrSz4HMDlZbZdkm90/SOJ2VMneYaTmdUMX3HdT57hZGa1xEmin2a96hiOmVjHrFcdU+1QzMxy5yTRT48+c4ADBw/x6DMHqh2KmVnunCT6afbUY5l0VB0EHpcwszHPSaKfxo0bR9048YW1W3noyeeqHY6ZWa6cJPqpcdpkPvLmU9j3YieP7H6+2uGYmeXKSaKfJDFjyiuyyyTc3WRmY5yThJmZleUkMQBdt+d48rkXPXhtZmOak8QAnHricUyaIFbcuZMtbX+qdjhmZrlxkhiA108/nvefM43nDx7izh27qx2OmVlunCQGQBInTjoagDhc5WDMzHLkJDFAdXV1AGz443McOnSoytGYmeXDSWKAlr1tNn8x+3h+0/oM19zx+2qHY2aWCyeJAaqrq2PWqyYB8PTel6ocjZlZPpwkBqFhSnYn2N8+8iwHDx6scjRmZkPPSWIQPva2U5k5ZSKPP/tvnH/Vr+ns7Kx2SGZmQyr3JCFpgaQdklolXVmiXJKuTuWbJc2tdNtqq6ur46/+/CQAntr/Mn+54m4PYpvZmJJrkpBUB1wDNAONwBJJjUXVmoE56bUUWNGPbavu4297LSedkE2HffDJ5znzi7dx8/2Pcviw58aa2eiX9zOu5wGtEbETQNIaYBGwraDOImB1etb1vZKmSGoATqlg26qrq6vj/33qLcz9+1/y/MHDHOiET9+yjc/eso1jxsOEOjhYohdKGvllIyUOl7nMZb2XjR8Pp08/gX++tImJEyf23GAQ8k4S04EnCpbbgDdVUGd6hdsiaSlZC4STTjpp8BEPwMSJE9n4+Xew6J/u5Pd7XuYw8DKwtxPobZhiNJSNlDhc5jKXlS87BHfv/BMfvfEBVv+X+b1s1H95JwmVWFd8R7xydSrZlohYBawCaGpqqtrd9o466ihu+/QFvPjii1zwrTt47oXDVf/rYrBlIyUOl7nMZb2XdbUkrr1obs/Kg5R3kmgDZhYszwB2VVhnYgXbjjhHH300d32uudphmJkNibxnN20A5kiaJWkisBhYW1RnLXBxmuU0H9gbEe0VbmtmZjnKtSUREZ2SlgPrgTrg+ojYKmlZKl8JrAMWAq3AAeCy3rbNM14zM+tOY+mhOU1NTdHS0lLtMMzMRhVJGyOiqVSZr7g2M7OynCTMzKwsJwkzMyvLScLMzMoaUwPXkjqAxwexi6nAniEKp5rGynmAz2WkGivnMlbOAwZ3LidHRH2pgjGVJAZLUku5Ef7RZKycB/hcRqqxci5j5Twgv3Nxd5OZmZXlJGFmZmU5SXS3qtoBDJGxch7gcxmpxsq5jJXzgJzOxWMSZmZWllsSZmZWlpOEmZmV5SQBSFogaYekVklXVjueSkh6TNIWSZsktaR1r5T0C0l/SO8nFNT/bDq/HZLeU73IQdL1knZLeqhgXb9jl/TG9G/QKulqSaUeVDXc5/ElSU+m72WTpIUj/TxSDDMl3SFpu6Stkj6Z1o+q76WX8xh134ukoyXdL+nBdC5fTuuH9zuJiJp+kd2G/BFgNtmDjh4EGqsdVwVxPwZMLVr3DeDK9PlK4Ovpc2M6r6OAWel866oY+/nAXOChwcQO3A+cS/YUw58DzSPgPL4EfLpE3RF7HimGBmBu+nwc8PsU86j6Xno5j1H3vaTjTkqfJwD3AfOH+ztxSwLmAa0RsTMiDgJrgEVVjmmgFgE3pM83AO8vWL8mIl6KiEfJnt0xb/jDy0TEXcCzRav7FbukBmByRNwT2U/B6oJthkWZ8yhnxJ4HQES0R8QD6fN+YDvZc+ZH1ffSy3mUMyLPAyAyz6fFCekVDPN34iSR/Q/0RMFyG73/TzVSBHC7pI2SlqZ1r47sqX6k9xPT+tFwjv2NfXr6XLx+JFguaXPqjurqChg15yHpFOAcsr9cR+33UnQeMAq/F0l1kjYBu4FfRMSwfydOElnzq9homBd8XkTMBZqByyWd30vd0XqOUD72kXpOK4BTgbOBduAf0/pRcR6SJgE/Bq6IiH29VS2xbsScT4nzGJXfS0QcioizgRlkrYIzeqmey7k4SWRZdWbB8gxgV5ViqVhE7Ervu4GfkHUfPZ2alqT33an6aDjH/sbelj4Xr6+qiHg6/WAfBr7LkW69EX8ekiaQ/WL9XkTcklaPuu+l1HmM5u8FICKeA34NLGCYvxMnCdgAzJE0S9JEYDGwtsox9UrSsZKO6/oMXAA8RBb3JanaJcD/TZ/XAoslHSVpFjCHbCBrJOlX7KmZvV/S/DRT4+KCbaqm64c3+Y9k3wuM8PNIx74O2B4R3yooGlXfS7nzGI3fi6R6SVPS51cA7wIeZri/k+EcrR+pL2Ah2SyIR4DPVzueCuKdTTaL4UFga1fMwKuAXwJ/SO+vLNjm8+n8dlCF2TNF8f+ArMn/MtlfOR8ZSOxAE9kP+yPA/yLdQaDK5/GvwBZgc/qhbRjp55FieDNZF8RmYFN6LRxt30sv5zHqvhfgDcDvUswPAV9I64f1O/FtOczMrCx3N5mZWVlOEmZmVpaThJmZleUkYWZmZTlJmJlZWU4SZkNE0hWSjql2HGZDyVNgzYaIpMeApojYU+1YzIaKWxJmA5Cuev9Zutf/Q5K+CEwD7pB0R6pzgaR7JD0g6UfpfkJdzwL5enpWwP2SXpvWfyDt60FJd1Xv7MyOcJIwG5gFwK6IOCsizgC+TXY/nLdHxNslTQX+DnhXZDdibAH+umD7fRExj+zq12+ndV8A3hMRZwHvG57TMOudk4TZwGwB3pVaBG+JiL1F5fPJHgJzd7rV8yXAyQXlPyh4Pzd9vhv4F0n/lexhWGZVN77aAZiNRhHxe0lvJLsv0Nck3V5URWT3/19SbhfFnyNimaQ3Ae8FNkk6OyKeGerYzfrDLQmzAZA0DTgQETcC3yR7jOl+skdmAtwLnFcw3nCMpD8r2MUHC97vSXVOjYj7IuILwB663/bZrCrckjAbmDOBqyQdJrsL7MfIuo1+Lqk9jUtcCvxA0lFpm78ju9swwFGS7iP7Q62rtXGVpDlkrZBfkt3l16yqPAXWbJh5qqyNJu5uMjOzstySMDOzstySMDOzspwkzMysLCcJMzMry0nCzMzKcpIwM7Oy/j809Ewp1E5MPAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_scatter(list(range(3001)), ys, 'steps', 'y_inits')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a6822f5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "3dfd0581",
   "metadata": {},
   "source": [
    "分割线\n",
    "\n",
    "------------\n",
    "\n",
    "\n",
    "\n",
    "使用PINN解决高维PDE问题，待做"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6ddd542",
   "metadata": {},
   "source": [
    "100维 Allen-Cahn 方程\n",
    "\n",
    "$$\n",
    "\\large  \\frac{\\partial u}{\\partial t}(t,x) - \\Delta u(t,x) - u(t,x) + [u(t,x)]^3, t\\in [0,T],x\\in \\mathbb{R}^{100},\\\\\n",
    "\\large  u(0,x) = g(x),\\\\\n",
    "\\large  g(x) = \\frac{1}{2+0.4||x||^2}.\\\\\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "818f04ea",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: tensorflow.compat.v1\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From G:\\Anaconda3\\envs\\py3.8\\lib\\site-packages\\tensorflow\\python\\compat\\v2_compat.py:101: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "non-resource variables are not supported in the long term\n",
      "WARNING:tensorflow:From G:\\Anaconda3\\envs\\py3.8\\lib\\site-packages\\deepxde\\nn\\initializers.py:118: The name tf.keras.initializers.he_normal is deprecated. Please use tf.compat.v1.keras.initializers.he_normal instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import deepxde as dde\n",
    "from deepxde.backend import tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fe2ea44",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_dim = 100\n",
    "\n",
    "T = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de8ed9d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "geom = dde.geometry.Interval(-1, 1) \n",
    "# deepxde.geometry.geometry_nd.Hypercube(xmin, xmax)\n",
    "# deepxde.geometry.geometry_nd.Hypersphere(center, radius)\n",
    "timedomain = dde.geometry.TimeDomain(0, T)\n",
    "geomtime = dde.geometry.GeometryXTime(geom, timedomain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1308b83",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pde(x, y):\n",
    "    dy_t = dde.grad.jacobian(y, x, i=0, j=1)\n",
    "    dy_xx = dde.grad.hessian(y, x, i=0, j=0)\n",
    "    return dy_t - dy_xx - y + y**3\n",
    "\n",
    "\n",
    "def pde(x, y):\n",
    "    dy_x = dde.grad.jacobian(y, x, i=0, j=0)\n",
    "    dy_t = dde.grad.jacobian(y, x, i=0, j=1)\n",
    "    dy_xx = dde.grad.hessian(y, x, i=0, j=0)\n",
    "    return dy_t + y * dy_x - 0.01 / np.pi * dy_xx\n",
    "\n",
    "def pde(x, y):\n",
    "    dy_x = tf.gradients(y, x)[0]\n",
    "    dy_x, dy_t = dy_x[:, 0:1], dy_x[:, 1:]\n",
    "    dy_xx = tf.gradients(dy_x, x)[0][:, 0:1]\n",
    "    dy_xxx = tf.gradients(dy_xx, x)[0][:, 0:1]\n",
    "    return dy_t + lambda_1*y*dy_x +  lambda_2*dy_xxx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b163d77",
   "metadata": {},
   "outputs": [],
   "source": [
    "ic = dde.icbc.IC(geomtime, \n",
    "                 lambda x: 1.0/(2.0+ 0.4*tf.reduce_sum(tf.square(x[:, 0:n_dim]), 1, keepdims=True)), \n",
    "                 lambda _, on_initial: on_initial)\n",
    "\n",
    "data = dde.data.TimePDE(geomtime, pde, [ic],\n",
    "                        num_domain=2540, num_boundary=80, num_initial=160)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40f94e08",
   "metadata": {},
   "outputs": [],
   "source": [
    "net = dde.nn.FNN([n_dim+1] + [200] * 3 + [1], \"tanh\", \"Glorot normal\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8846fd8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = dde.Model(data, net)\n",
    "model.compile(\"adam\", lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0456bfb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "## https://github.com/frankhan91/DeepBSDE/blob/master/equation.py\n",
    "\n",
    "class AllenCahn():\n",
    "    \"\"\"Allen-Cahn equation in PNAS paper doi.org/10.1073/pnas.1718942115\"\"\"\n",
    "    def __init__(self):\n",
    "        self.dim = 100\n",
    "        self.total_time = 0.3\n",
    "        self.num_time_interval = 20\n",
    "        self.delta_t = self.total_time / self.num_time_interval\n",
    "        self.sqrt_delta_t = np.sqrt(self.delta_t)\n",
    "        self.y_init = None\n",
    "        # ---------------------------\n",
    "        self.x_init = np.zeros(self.dim)\n",
    "        self.sigma = np.sqrt(2.0)\n",
    "\n",
    "    def sample(self, num_sample):\n",
    "        \"\"\"Sample forward SDE.\"\"\"\n",
    "        dw_sample = np.random.normal(size=[num_sample, self.dim, self.num_time_interval]) * self.sqrt_delta_t\n",
    "        x_sample = np.zeros([num_sample, self.dim, self.num_time_interval + 1])\n",
    "        x_sample[:, :, 0] = np.ones([num_sample, self.dim]) * self.x_init\n",
    "        for i in range(self.num_time_interval):\n",
    "            x_sample[:, :, i + 1] = x_sample[:, :, i] + self.sigma * dw_sample[:, :, i]\n",
    "        return dw_sample, x_sample\n",
    "\n",
    "    def f_tf(self, t, x, y, z):\n",
    "        \"\"\"Generator function in the PDE.\"\"\"\n",
    "        return y - tf.pow(y, 3)\n",
    "\n",
    "    def g_tf(self, t, x):\n",
    "        \"\"\"Terminal condition of the PDE.\"\"\"\n",
    "        return 0.5 / (1 + 0.2 * tf.reduce_sum(tf.square(x), 1, keepdims=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37c61500",
   "metadata": {},
   "outputs": [],
   "source": [
    "losshistory, train_state = model.train(epochs=15000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2125143d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dc91780",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "243ba90a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f76eb44e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.8",
   "language": "python",
   "name": "py3.8"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
